Regularization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iris = datasets.load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = iris[\"data\"][:, 3:]  # petal width"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = (iris[\"target\"] == 2).astype(np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1d51b1ddbe0>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4Tdf+x/H3SiRIiClJEbNGzYqYStVU8yzUWDVWi7Z6\nq9z+tHr16uC2OhqqhppaMzVTSilKElPMIoYIKuYpEknW748VpIocnGSfc/J9Pc9+cobtnM/u1m+W\ntddeS2mtEUII4VrcrA4ghBDC/qS4CyGEC5LiLoQQLkiKuxBCuCAp7kII4YKkuAshhAuS4i6EEC5I\nirsQQrggKe5CCOGCMln1xb6+vrpIkSJWfb0QQjilsLCwc1prv9T2s6y4FylShNDQUKu+XgghnJJS\n6rgt+0m3jBBCuCAp7kII4YJSLe5KqclKqbNKqT0PeF8ppb5RSkUopXYrpSrZP6YQQohHYUvL/Ueg\n8UPebwIEJm99gXFPHksIIcSTSLW4a603ABcesksrYJo2/gRyKqXy2SugEEKIR2ePPvcAICrF85PJ\nr/2DUqqvUipUKRUaExNjh68WQghxP+l6QVVrPUFrHaS1DvLzS3WYphBCiMdkj3Hu0UDBFM8LJL+W\nZtauhQMHwN8f/PzMT39/yJMHlErLbxZCCOdgj+K+GBiglJoFVAMua61P2+FzH2jWLJg48Z+vx8WB\npyd8+imsWWMKfv78EBAABQpAu3bg5gZayy8BIYRrS7W4K6V+BuoAvkqpk8BwwANAaz0eWA40BSKA\nG0CPtAp727hx8N//QkwMnD1rtosXTWEHcHeH2FjYtg1OnTKPvbwgONi837MnrFplCn6BAlC4MDz9\nNLz+uin6UvyFEGkh9lYsRy4eIbtndgrnLJym36W01mn6BQ8SFBSk02P6Aa1N4T97FkqWNK9NnQob\nNsDJk2Y7dgxy5TKPAdq2hdBQKFbMbIGBUL48NGuW5nGFEE7uVuItjl46yqHzhzh8/jCHLxw2jy8c\nJupyFBrNkJpD+LTBp4/1+UqpMK11UKr7uXpxt4XWcOmSKfAA338PmzZBZKTZTp+G0qVh717zft++\ncOUKlCplXq9QwbT83eR+XyEyDK01Jy6fIPxsOOF/hbMnZg/hf4Vz4NwBbiXdurNfziw5KZGnBIG5\nA+/8DMofRGCewMf6XluLu2UThzkSpe4WdoBXXzXbbVeumJb/bbduQUgIzJljfjEAvPACrF9vHi9b\nBoUKmeKfSf4LC+H0knQSh88fJuRUCCHRIYSdDiP8bDhX4q7c2adQjkKU8y9H08CmlPYrTWDuQALz\nBJInax6UBf28Unps4ONjttumTDE/b9wwo3Z27gRvb/NaYiK0b2/6+bNkgUqVoGZNaNECnn8+/bML\nIR5dzPUY/jjxB1ujtxJyKoTQU6F3CrmXhxcV81akW/lulPUvSzn/cpT1L0uOLDksTv13UtyfgJeX\nKd6VUsymoxSEhcH27WbbsgW+/tq08J9/Hm7eNBdua9SAevWgeHHr8gshjKjLUWw4voENxzew8cRG\n9p/bD4CHmwcV8lagS7kuVMlfhSoBVSjpW5JMbo5fOqXPPR3cvGla8rlymZZ+7dpmpA9AkSLQoIEp\n+BUrWhpTiAzj0s1LrI1cy8qIlaw5uoZjl44BkCNzDmoWqkntQrWpXbg2lfJVInOmzNaGvYf0uTuQ\nLFnMBmbEzl9/wcGD5masNWtg7lwzQgdg1y7zvGVLCAqSi7RC2EOSTiLsVBgrI1ay6sgq/jz5J4k6\nEZ/MPtQrWo9B1QdRu3BtyvmXw93N3eq4diEtdweQkGB+ZsoEEyaYVnxiIuTNC82bm0LfqNHdcfxC\niNTdSrzF+mPrWXhgIYsOLOL0tdMoFJXzV6ZR8UY0frox1QKq4eHuYXXURyJDIZ3YhQuwYgUsXmx+\n3rxpunFy5ICjRyFfvrv/EhBC3BWXEMeKiBUs2L+AJYeWcOnmJbw8vGjydBNal2xNo+KN8PN27nmt\npFvGieXODV26mC0+HnbvNoUdzGt795punI4doX59GW4pMrYkncSmE5uYsXsGc/bN4dLNS+TKkouW\nz7Skbcm2NCzekKweWa2Ome6kLDg4T0/T937b8OFmbp0FC+DHH03XzXvvwcCBlkUUwhIRFyL4ceeP\nzAyfybFLx/Dy8KJNyTZ0Ld+V+kXrO113i71JcXcyjRqZbdw402UzZcrdPvsbN2DhQtOqz5rxGioi\nA7iVeIvFBxczPmw8ayLX4KbcaFCsASPqjKBNqTZk88xmdUSHIcXdSWXJAm3amO22pUuha1fImRN6\n94YBA8ykaEI4uxOXTzAhbAKTdkzizLUzFPQpyEd1P6JnxZ7kz57f6ngOSYq7C2nfHp56CsaOhS+/\nhNGjTfH/4Ye/T68ghLMIOxXG51s+Z+7euSTpJJqVaEa/yv1o/HRjlxmymFakuLsQpcwcNy+8AFFR\nMGYM/P773YuxBw6YGS7d5f8J4cCSdBIrDq/g8y2fs/7YerJ7Zuet6m8xsOrANJ8m15VIcXdRBQua\nRUtuz01/7ZqZ48bf31yA7dRJRtkIx5KYlMjcfXP5aMNH7IvZRwGfAnz+4uf0rtTb4eZtcQZy/6OL\nuz0ZXdasMH68GX3z8stQooS5YerWrYf/eSHSWpJOYvae2ZQbV45O8zsBML3NdCLfiORfz/1LCvtj\nkuKeQbi7mz75nTvNzVF+fmZa47VrrU4mMiqtNfP2zaPcuHJ0nN8RpRSzg2cT/lo4Xct3zfBDGZ+U\n/MM8g1HKTD/cvDls3Hh3GuIJE8zImoYNZYlBkfY2ndjEO7++w58n/6SUbylmtZtFcOlguUhqR9Jy\nz6CUMrNTKmXmsfn2W2jc2MxQGR5udTrhqg6fP0y7Oe2oNaUWJy6fYFLLSYS/Fs5LZV+Swm5nUtwF\n7u5mDvpvvjGzUj77rLnj9cIFq5MJV3H55mXeWvkWpceWZvWR1YyoM4JDAw7Rs2JPKeppRIq7AMyF\n1oED4dAheO01cwfsnj1WpxLOTmvNT+E/UXJMSb7Z+g09n+1JxMAI3n/hfbw9va2O59KkuIu/yZ0b\nvvvOLAxeu7Z5bcwY2LHD2lzC+eyP2U/9afXpsqALBX0Ksq3PNr5v8T1PZXvK6mgZghR3cV+FCpmf\n16/DJ59AlSowZIhZUUqIh4lLiGPYb8MoP748O8/sZFyzcWzptYWg/KnOUivsSIq7eChvb3OBtUcP\nGDUKypWDdeusTiUcVeipUCpPqMzIjSPpXK4zBwYcoF9QP+lXt4AUd5GqXLnM/DS//WZG17z4olk0\nRIjb4hLieG/te1SfWJ1LNy+xrPMypraeir+3v9XRMiwZ5y5sVreuWTjk99+haFHzWlSUmepAZFxh\np8J4edHL7IvZR49nezC60WhyZslpdawMT1ru4pFkzWrGw4NpyRcrZhYQkWkMMp4kncTnmz+nxqQa\nXL55meWdlzO51WQp7A5Cirt4bJUrQ+fOMGKEmZRMumoyjtNXT9N4RmMG/zqY5iWas/u13TQJbGJ1\nLJGCFHfx2HLkgKlTYe5cMz6+YkVYtMjqVCKtLT+8nArjK7DxxEbGNxvP/A7zyZ01t9WxxD2kuIsn\nFhxsxsEHBsK+fVanEWklMSmR/1v7fzT7qRl5s+UltE8orwa9ipLJiBySXFAVdlG0KPzxB3gkT+S3\ncaMZKy/L/LmGczfO0Wl+J9ZErqFXxV582+RbsnrIQr2OzKaWu1KqsVLqoFIqQik19D7v51BKLVFK\n7VJK7VVK9bB/VOHoMmcGNzezYPcrr0BQkBlZI5xbSHQIlb6vxMbjG/mhxQ9MbDlRCrsTSLW4K6Xc\ngTFAE6A00EkpVfqe3foD+7TWFYA6wBdKKU87ZxVOIlMmWLEC8uQxs0yOHWtWhBLOZ/KOydSaUgs3\n5camnpvoXam31ZGEjWxpuVcFIrTWkVrreGAW0OqefTSQXZnOt2zABSDBrkmFUylRArZuhUaNoH9/\nszBIfLzVqYStEpMSeWf1O/Ra3IsXCr9AWN8wKuevbHUs8Qhs6XMPAKJSPD8JVLtnn++AxcApIDvw\nktY6yS4JhdPKkQN++QU++MCsACULczuHq3FX6bygM0sPLWVAlQF82fhLMrnJ5TlnY68z1gjYCdQD\nigO/KqU2aq2vpNxJKdUX6AtQ6PbMVMKlubvDyJGmH97dHc6cgatXzcga4XhOXD5Bi59bsPfsXr5r\n8h39q/a3OpJ4TLZ0y0QDKW8wL5D8Wko9gAXaiACOAiXv/SCt9QStdZDWOsjPz+9xMwsnlCm5GdGj\nB9SoAZs3W5tH/FNIdAhVf6jKsUvHWNZ5mRR2J2dLcQ8BApVSRZMvknbEdMGkdAKoD6CUegp4Boi0\nZ1DhGr791swZX68ezJtndRpx28qIldSZWgcvDy+29NpCo6cbWR1JPKFUi7vWOgEYAKwC9gNztNZ7\nlVL9lFL9knf7CHhOKRUOrAWGaK3PpVVo4byeftq02itXhg4dYPRoqxOJ6bum0+LnFpTIU4LNvTZT\n2u/ewXDCGSlt0Ri1oKAgHRoaasl3C+vFxkK3brBtm5lpMqfMNWWJzzd/zuBfB1O3SF0WdVyET2Yf\nqyOJVCilwrTWqa58IpfAhSWyZoXZs+Gvv0xhT0oyY+FlRE36SNJJvPvru3yx5Qval27P9DbTyZwp\ns9WxhB3J3DLCMu7ukD+/Ker9+8PLL8vUwekhMSmRPov78MWWL+hfpT8/t/tZCrsLkuIuLKeUmYPm\np5+gbVtZpzUtJSQl0H1RdybvnMwHtT/g2ybfyhJ4LkqKu3AIQ4fCuHGwbBk0bWoW5hb2FZ8YT6f5\nnZgZPpOP633Mf+r+R2Z0dGFS3IXD6NcPZsyADRugZUuZj8ae4hLiCJ4TzLx98/iy0Zf8+/l/Wx1J\npDG5oCocSufOppvGzc38FE8u9lYsbWa3YdWRVYxtOpbXqrxmdSSRDqS4C4fTqdPdx+vXm6mDs2Wz\nLI5Ti0uIo+2ctqw+sppJLSfRs2JPqyOJdCLdMsJhnTkDTZpAs2Zw7ZrVaZxPfGI8HeZ1YGXESn5o\n8YMU9gxGirtwWHnzwpQpZoWn5s1lFM2jSEhKoMuCLiw+uJgxTcfQq1IvqyOJdCbFXTi0jh1h+nRz\nkTU4WOaEt0ViUiLdF3Vn3r55jG44mtervG51JGEBKe7C4XXuDOPHw/Ll8MMPVqdxbEk6iT5L+vBT\n+E98XO9jBtUYZHUkYRG5oCqcQt++UKQI1K9vdRLHpbXmrZVvMWXnFD6o/YEMd8zgpOUunEbDhmbK\ngqgoGDVKxsHfa+TGkXy77VsGVR/Eh3U+tDqOsJgUd+F0pkyBIUNgxAirkziO70O/5/1179OtfDc+\nb/i53HkqpFtGOJ9hw+DoUfjwQzOi5tVXrU5krfn75vP68tdpGtiUSS0n4aakzSakuAsn5OZmLqye\nPQuvv24KfKtWVqeyxrqj6+i8oDPVC1Rnbvu5eLh7WB1JOAj5FS+cUqZMMGeOuXv1gw8gMdHqROlv\n++nttJrVisDcgSzptAQvDy+rIwkHIi134bS8vWHpUvM4oy3ycfTiUZrMbEKurLlY1XUVubPmtjqS\ncDDSchdOzc/PbPHx8O67cPq01YnS3sXYizT9qSnxifGs6rqKAJ8AqyMJByTFXbiEiAgYO9bMBX/l\nitVp0k58Yjzt5rTjyIUjLHppESV9S1odSTgoKe7CJZQuDfPmQXg4vPQSJCRYncj+tNb0WdKHdcfW\nMbnVZF4o8oLVkYQDk+IuXEbjxqb1vnIlDHLBu+5H/D6Cabum8Z86/6Fr+a5WxxEOToq7cCl9+8Lb\nb8PEiaarxlVM2zWND3//kO4VuvN+7fetjiOcgBR34XJGjYKwMHj6aauT2Mf6Y+vpvbg3dYvUZUKL\nCXL3qbCJFHfhctzdTR88wLRpsGePtXmeRMSFCNrObsvTuZ9mwUsL8HT3tDqScBIyzl24rKtXYehQ\n8PSErVvhqaesTvRorsRdoeXPLVFKsaTTEnJmyWl1JOFEpOUuXFb27LB4sZmmoHVruHnT6kS2S0xK\npPP8zhw6f4i57edSPHdxqyMJJyPFXbi0oCCzktOff5oJxpxlmuD31r7HssPL+KbJN9QrWs/qOMIJ\nSXEXLq9dOzOD5LRpsGaN1WlSN33XdEZtHkW/yv1kiTzx2KTPXWQI778PVapAgwZWJ3m4rSe30mdJ\nH+oUqcM3Tb6xOo5wYtJyFxmCm5uZmkApM3omMtLqRP8UfSWa1rNbkz97fpm+VzwxabmLDCU+Hpo1\nAx8f2LIFsmWzOpEReyuW1rNbcy3+Gmu6rcHXy9fqSMLJ2dRyV0o1VkodVEpFKKWGPmCfOkqpnUqp\nvUqp3+0bUwj78PQ0d6/u2wevvAJJSVYnMgauGEjoqVBmtp1JGf8yVscRLiDV4q6UcgfGAE2A0kAn\npVTpe/bJCYwFWmqtywDt0yCrEHbx4ovwv//B/PkwcqTVaWDS9klM2jGJYc8Po+UzLa2OI1yELS33\nqkCE1jpSax0PzALuXdSsM7BAa30CQGt91r4xhbCvQYOga1ezitOvv1qXY8fpHfRf3p8GxRrwYZ0P\nrQsiXI4tfe4BQFSK5yeBavfsUwLwUEqtB7IDX2utp9kloRBpQCmYMAGKFIGaNa3JcDH2IsFzg/Hz\n9uOntj/h7pbBlpMSacpeF1QzAZWB+kBWYItS6k+t9aGUOyml+gJ9AQoVKmSnrxbi8WTNCh99ZB5f\nu2Z+ptcF1iSdRPdF3Ym6HMWGHhvw8/ZLny8WGYYt3TLRQMEUzwskv5bSSWCV1vq61vocsAGocO8H\naa0naK2DtNZBfn7yl1k4hvh4eO456N07/e5g/eyPz1hyaAmjG42meoHq6fOlIkOxpbiHAIFKqaJK\nKU+gI7D4nn1+AWoppTIppbww3Tb77RtViLTh6QmdOsHs2fBNOtw3tDZyLcPWDaNT2U70r9I/7b9Q\nZEipdstorROUUgOAVYA7MFlrvVcp1S/5/fFa6/1KqZXAbiAJmKi1duKJVkVGM2SImTnynXegcmWo\nVSttvif6SjSd5neipG9JmZtdpCmlLZpJKSgoSIeGhlry3ULcz+XLZqKxa9dg+3bIl8++nx+fGE+d\nH+sQfjackD4hsri1eCxKqTCtdVBq+8n0A0Iky5EDFiyAgIC7F1jt6d1f32XLyS1MajlJCrtIczL9\ngBAplCsHISFmqKQ9zd4zm6+3fs1b1d6iQ5kO9v1wIe5DWu5C3EMpiI2F7t1hzpwn/7z9MfvptbgX\nNQvWZNSLo578A4WwgRR3Ie7D3R0OH4aePc08NI/rWvw12s1ph7enN7ODZ8tMjyLdSHEX4j48PWHu\nXPD2hrZtzXqsj0prTZ8lfTh4/iCz2s0iwCfA/kGFeAAp7kI8QECAGfseEQG9ej36DU7fbfuOWXtm\nMbLeSOoWrZs2IYV4ACnuQjxEnTrw8cewdCkcOGD7n9sStYW3V79Ny2da8m7Nd9MsnxAPIsVdiFQM\nHmxWbypVyrb9z14/S/u57SmUoxBTW0/FTcn/ZiL9yd86IVKhFBQrZh5Pnw5nzjx438SkRDrP78z5\n2PPM7zCfnFlypk9IIe4hxV0IG0VFwauvmnloEhLuv8/w9cNZe3QtY5uO5dm8z6ZvQCFSkOIuhI0K\nFoTx42H9ehg27J/vLz20lJEbR9K7Ym96VOyR7vmESEmKuxCP4OWXTev9s8/gl1/uvh55MZJuC7tR\nKV8lvm36rXUBhUgmxV2IR/TVV2aCse7d4exZiL0VS/CcYADmtZ9HlkxZLE4ohMwtI8Qjy5LF3OC0\nYQP4+0PvxQPZcWYHSzstpWiuolbHEwKQ4i7EYylSxGyTd0xm0vpVvNfs/2hWopnVsYS4Q4q7EI9p\nx+kdvDbpe9x+OEzhkp5mBWEhHIT0uQvxGC7GXiR4bjB+Rc9Q+3k33njDjbAwq1MJcZcUdyEeUZJO\novui7kRdjmLeS7OZO8sTf38IDoYLF6xOJ4QhxV2IR/TZH5+x5NASvmj4BdULVMfX11xgjY6Gbt0g\nKcnqhEJIcRfikayNXMuwdcPoWLYjA6oOuPN6tWpmiKSfH9y6ZWFAIZLJAtlC2Cj6SjQVv6+Ir5cv\n2/psI5tntr+9f/t/JXsv0SdESrJAthB2FJ8YT/u57YlNiGXBSwv+UdjBFHWlYP9+eP55MxeNEFaR\n4i6EDd799V22nNzCpJaTKOlb8qH7urnBrl3Qvj3Ex6dTQCHuIcVdiFTM3jObr7d+zZvV3qRDmQ6p\n7v/MMzB5MmzdCv/6VzoEFOI+pLgL8RD7Y/bTa3Evniv4HKNeHGXznwsOhrffhu++g59/TsOAQjyA\nFHchHuBq3FXazmmLl4cXc4Ln4Onu+Uh//tNPoVYtM4pGhkeK9CbTDwhxH1prei7uyaHzh1jTbQ0B\nPgGP/BkeHjB/Pnh5mX54IdKT/JUT4j5GbxnNvH3z+LT+p9QtWvexP8ffH7Jlg+vXTReNRSOPRQYk\nxV2Ie/x+7HeGrBlC21Jteee5d+zymTNnwsCB8PXXdvk4IVIlxV2IFKKvRNNhXgeezv00U1pNQdnp\njqQ+faB1axg8GDZtsstHCvFQUtyFSBafGE+HeR24Hn+d+R3m45PZx26frRT8+KOZA75DB/jrL7t9\ntBD3JcVdiGSDVw9mc9RmJrWcRBn/Mnb//Bw5zAXWixehd2+7f7wQf2NTcVdKNVZKHVRKRSilhj5k\nvypKqQSlVLD9IgqR9n4K/4lvtn3DoOqDeKnsS2n2PeXLm/73Tz9Ns68QArChuCul3IExQBOgNNBJ\nKVX6Aft9Bqy2d0gh0lL4X+H0WdKHWoVq8VmDz9L8+9q0gTJlzMiZ48fT/OtEBmVLy70qEKG1jtRa\nxwOzgFb32W8gMB84a8d8QqSpyzcv03ZOW3wy+zAneA4e7h7p9t2ffAIVKsCRI+n2lSIDsaW4BwAp\n57c7mfzaHUqpAKANMO5hH6SU6quUClVKhcbExDxqViHsKkkn0W1hN45dOsbc9nPJlz1fun5/587m\n5qZ27SA2Nl2/WmQA9rqg+hUwRGv90JustdYTtNZBWusgPz8/O321EI9n+LrhLDm0hNENR1OrUK10\n//4iRUz/++7d8PrrcoOTsC9bins0UDDF8wLJr6UUBMxSSh0DgoGxSqnWdkkoRBqYv28+/934X3o+\n2/NvKyqltyZNYNgwM0xy0iTLYggXZMvcMiFAoFKqKKaodwQ6p9xBa1309mOl1I/AUq31IjvmFMJu\nwv8Kp/ui7lQvUJ2xzcba7UalxzV8uGm9Z8liaQzhYlIt7lrrBKXUAGAV4A5M1lrvVUr1S35/fBpn\nFMJuzt84T6tZrfDJ7MP8DvPJnCmz1ZFwd4eFC+8uz6e1LNUnnpxNs0JqrZcDy+957b5FXWv9ypPH\nEsL+EpIS6DCvA9FXo9nwygbyZ89vdaQ7bhfzmTPhhx9g5UppyYsnI3eoigxj8OrB/Hb0N8Y3G0+1\nAtWsjnNfXl7w++/Qt69cYBVPRoq7yBCm7ZrGV1u/4o2qb9CjYg+r4zxQmzYwYgRMnw6ff251GuHM\npLgLl7f15Fb6LulL3SJ1+byh41fMYcPM4tpDhsCyZVanEc5KirtwaccvHaflrJYE+AQwp3363oH6\nuG7PIPnss/Dbb1anEc5KltkTLutK3BWa/9ycuIQ41ndfj6+Xr9WRbOblBRs2gLe31UmEs5KWu3BJ\nCUkJdJzXkf0x+5nXYR6l/EpZHemRZctmWvF79kCPHhAfb3Ui4Uyk5S5c0tur3mZFxArGNxtPg2IN\nrI7zRHbuNN00WsOUKTIGXthGirtwOWO2jeHbbd8yqPogXg161eo4T6xrVzNz5IcfQvHi8P77VicS\nzkCKu3ApKyNW8sbKN2hRogX/e/F/Vsexmw8+gMhI87NoUVPwhXgY6XMXLmPXmV10mNuBcv7l+Knd\nT7i7uVsdyW6UMneu1q0LY8dC0kPnXxVCWu7CRRy/dJwmM5vgk9mHJZ2WkM0zm9WR7M7TExYsAA8P\nMw+8EA8jf0WE07sQe4EmM5tw49YNVnZdScEcBVP/Q04qZ04zPPLaNejXD87KumfiAaS4C6cWeyuW\nlj+35MjFI/zS8RfK+pe1OlK6OHwYpk2DZs3g6lWr0whHJMVdOK3EpES6LOjC5qjNTG8znReKvGB1\npHRTsSLMng07dkDbtjIGXvyTFHfhlLTWvLHiDRYeWMiXjb6kQ5kOVkdKdy1awMSJsGYNvPyyXGQV\nfycXVIVT+njjx4wNHcs7Nd7hzepvWh3HMq+8AjExMHIkRERAiRJWJxKOQlruwul8t+07hq0bRpdy\nXfjsxc+sjmO5wYNh/34p7OLvpLgLpzJ151QGrhhIq2daMaXVFNyU/BUGyJfPTE/wyScwbpzVaYQj\nkG4Z4TQW7F9Az8U9qV+0PrOCZznF9L3pKSkJNm82c8Bnzy53sWZ00uwRTmFVxCo6zutItYBqLOq4\niCyZZIHRe7m7w5w5UKcOdO8Oc+danUhYSYq7cHh/nPiDNrPbUNqvNMs6L3PJu0/tJWtWWLIEnnsO\nOneGX36xOpGwihR34dA2R22m6cymFMxRkNXdVpMray6rIzk8b2/TNVO5splNUmRM0ucuHNbmqM00\nntGYvNny8tvLv+Hv7W91JKfh42NWcvL0NM9jY02rXmQc0nIXDillYV/XfR0BPgFWR3I6twt7SIiZ\nB17WY81YpLgLhyOF3b4KF4Y8ecw8NKtWWZ1GpBcp7sKhbDqxSQq7nfn7w7p1ULIktGxpLrgK1yfF\nXTiMX4/8SsMZDaWwpwFfX9MtU6GCmWjs99+tTiTSmhR34RDm75tPs5+aEZg7kI09NkphTwO5cplJ\nxt56C6pWtTqNSGtS3IXlpuyYQod5HQjKH8S67ut4KttTVkdyWT4+8L//mZEzFy/CjBlWJxJpRYq7\nsNRXf35Fz8U9aVCsAb92+1XGsaejL7+Ebt1g2DAzL41wLTLOXVhCa82w34bx8R8f065UO2a2nUnm\nTJmtjpWhfPABnD5tpgs+cwbGj4dMUhFchk0td6VUY6XUQaVUhFJq6H3e76KU2q2UCldKbVZKVbB/\nVOEq4hJXHeYBAAAPMklEQVTi6LawGx//8TG9K/ZmVvAsKewWyJQJJkwwLfdJk6BdO7hxw+pUwl5S\nLe5KKXdgDNAEKA10UkqVvme3o8ALWutywEfABHsHFa7hYuxFGs9szMzwmYysN5IJLSaQyU2ai1ZR\nCj76CL77ztzsJAtuuw5bWu5VgQitdaTWOh6YBbRKuYPWerPW+mLy0z+BAvaNKVzB8UvHqTm5JptO\nbGJGmxm89/x7KKWsjiWA/v3h4EEoUsRMHXz8uNWJxJOypbgHAFEpnp9Mfu1BegErniSUcD0h0SFU\nn1SdU1dPsarrKrqU72J1JHGP7NnNz88+M+PhV6+2No94MnYdLaOUqosp7kMe8H5fpVSoUio0JibG\nnl8tHNi0XdN4fsrzZHbPzKaem6hbtK7VkcRDdOlipixo2hTGjLE6jXhcthT3aKBgiucFkl/7G6VU\neWAi0Eprff5+H6S1nqC1DtJaB/n5+T1OXuFEEpIS+Neqf9F9UXdqFKxBaN9QyviXsTqWSEWhQvDH\nH6a4Dxhgumzi461OJR6VLcU9BAhUShVVSnkCHYHFKXdQShUCFgDdtNaH7B9TOJsLsRdoOrMpo/8c\nzcCqA1nddTW+Xr5WxxI2yp4dFi6Ed94xQyRDQqxOJB5VqsMUtNYJSqkBwCrAHZistd6rlOqX/P54\n4AMgDzA2+QJZgtY6KO1iC0e2/fR22s9tT9TlKCa2mEivSr2sjiQeg7u7uZu1Vy8z6RhAdDQEyMwQ\nTkFpi25NCwoK0qGhoZZ8t0gbWmvGhY5j0KpB+Hv7Myd4DjUK1rA6lrCT9euhUSP4+GN4+20zjFKk\nP6VUmC2NZ5l+QNjFlbgrdJrfif7L+1O/aH12vLpDCruLqVQJmjc3XTWtW8O5c1YnEg8jxV08sR2n\ndxA0IYh5++bxSf1PWNp5qfSvuyAfH5g3z8xJs3KlGS65bp3VqcSDSHEXjy0xKZFPNn5CtYnVuH7r\nOuu6r2NoraG4Kflr5aqUMlMG//mnuegq88I7LrnvWzyWyIuRvLzwZTZFbaJ96faMbz6e3FlzWx1L\npJOKFSEsDDInTwm0YQP4+UGpUtbmEndJE0s8Eq01k7ZPosL4Cuw5u4cZbWYwO3i2FPYMyNvbTD6m\ntRkPX7EijBoFCQlWJxMgxV08gsiLkTSc0ZDeS3pTJX8Vdr+2my7lu8j8MBmcUmaqgiZNYMgQqFkT\n9u2zOpWQ4i5SlZCUwBebv6Ds2LJsPbmVcc3GseblNRTKUcjqaMJB5M0LCxbAzz/DkSOmFS83PllL\n+tzFQ20/vZ2+S/oSdjqMls+0ZEzTMRTwkUk/xT8pBR07Qt26Zk6aSpXM62fPgr+/tdkyImm5i/s6\nd+Mc/Zb2I2hCECevnGRO8BwWvbRICrtI1VNPwYgR5g7XmBhzkbVrV7Pak0g/UtzF3yQkJfDdtu8I\n/DaQidsn8ma1Nzkw4ADty7SXvnXxyLJlMxOPzZ0LzzxjxsjLJGTpQ4q7uGNN5BoqfV+JgSsGUjlf\nZXa/tpsvG39Jziw5rY4mnFTWrKYVv2cP1Khhpi0oVUpWfEoP0ucuCD0VytA1Q1l7dC1FchZhQYcF\ntC7ZWlrqwm4CA2HFCli1ChYvNmPiQSYiS0vScs/ADp47SPu57anyQxV2/bWLrxp9xYH+B2hTqo0U\ndmF3SkHjxjB2rHkcFWWKfuvWsHOn1elcjxT3DOjAuQN0X9SdMmPLsDJiJcNfGE7kG5G8Wf1NMmfK\nbHU8kUHkzg3//reZbbJiRWjbFnbtsjqV65ApfzOQ3X/tZuTGkczdO5csmbLQL6gfQ2sNxd9bxqkJ\n61y6BF9/bS62XrkChw9D8eJWp3Jctk75K33uLk5rzYbjG/hiyxcsObSE7J7ZGVprKIOqD8LPW5Y6\nFNbLmROGD4c334Rly+4W9s8+g7JlzZ2vbtLH8MikuLuomwk3mbVnFl9v/ZqdZ3aSO2tuRtQZwYCq\nA8iVNZfV8YT4h5w5zeLcADdvwvffw9GjZnTN22+bsfJZslib0ZnI70MXE30lmuHrhlP4q8L0+KUH\ntxJvMaH5BKIGRfH+C+9LYRdOIUsWOHgQZswwM0/26QMFCsAvv1idzHlIy90F3Eq8xdJDS5m0YxIr\nIlagtaZZiWa8We1N6hetLyNfhFPy8DAt+c6dzaIgY8ea0TUA27dDZCS0amX2E/8kxd2JHTh3gMk7\nJjN111TOXj9L/uz5GVpzKD0r9qR4brkiJVyDUlCvntlu++EHGD/ejJfv1Ml02QQFybquKcloGSdz\n7NIxZu+Zzay9s9h5ZieZ3DLRvERzelfsTaOnG5HJTX5fC9eXmGhuipo6FZYsgbg4cwfspk2uX+Bl\ntIwLibocxcIDC5m1ZxZbTm4BoHqB6nzV6CteKvsSebPltTihEOnL3d0s1t28uRlKOW8eXLtmCrvW\nZsx8tWrQpo2Z0yYjkpa7A9Jas/30dhYfXMziQ4vZecbcvvds3mfpWKYjHcp0oGiuohanFMIxnT8P\nTZvCtm3meenSpsh37363z96ZScvdyVyMvci6Y+tYfWQ1Sw4t4dTVU7gpN2oWrMmoBqNo+UxLnvHN\noE0QIR5BnjywdauZ3mDRIli4ED75xBT2wEA4edLcFduwoWvPMy/F3SI3E26y6cQm1kSuYc3RNYSd\nCkOjyeaZjUbFG9HymZY0DWyKr5ev1VGFcEoFC8LAgWY7d+7uYt5Ll8Jrr5nHlSpBo0ZQpw7Uru1a\n4+ilWyadnLtxjs1Rm9kctZlNUZsIiQ4hLjGOTG6ZqF6gOg2KNqBBsQZUDaiKh7uM7RIirSQlmaGU\nq1bBypWwZYu5QHvihPmF8McfpmunVi3zrwBHY2u3jBT3NBCXEMfemL2EnQpjy8ktbIraxKHzhwDw\ncPOgUr5K1CxYk3pF61G7cG2yZ85ucWIhMq5r1yA01LTewQytnDXLPH7mGahSxWwDBzrGSBwp7unk\nWvw1dv+1m+2nt7Pj9A62n9nOnrN7SEhKACBP1jw8V/A5niv4HDUL1iQofxBZPbJanFoI8SBxcWZx\n740bTd99SAh4eZkJzcBMhXD1KpQvD+XKmS09W/hyQdXOLt+8zP5z+9kXs+/Otv/cfo5dOnZnH18v\nXyrlq8Q7Nd6hUr5KVMxXkeK5issdokI4kcyZTZdMrVp3X7t8+e7jmBhYvhwmTrz7WsOGppsHzHu+\nvubibS4LZ/uQ4p7ChdgLRF6M5MiFI+bnRfPz0PlDRF+NvrNfZvfMlPQtSY0CNej5bE+ezfssFfNV\nJCB7gBRyIVxQjhx3H0+fbsbSnz4N4eFm8/Ex72ltpku4/csgTx4oUcIMxRw82Lx28GD6jL3PMMU9\nLiGOU1dPcfLKyb9t0VejOXrpKJEXI7l089Lf/sxT3k9RLFcx6hWtRxm/MpT2K01pv9IUyVkEdzd3\ni45ECGE1pSB/frM1avT397ZuNQX80CHTlXP4sJmnHiAhAXr0gM2b0z6jUxf3W4m3iLkRw9nrZ+9s\nMdfvPv/r+l93CnrMjZh//Pnsntkp4FOAwjkLU6NADYrlKkbxXMUplqsYxXIVw9vT24KjEkI4K6VM\nq/xBLfOkJLMoSXqwqbgrpRoDXwPuwESt9af3vK+S328K3ABe0Vpvt3NWAFYcXsGgVYM4e/0sF29e\nvO8+Hm4e+Hv74+ftR0D2AKrkr0IBnwJ/2wJ8AvDJ7JMWEYUQ4r48Pc20COkh1eKulHIHxgAvAieB\nEKXUYq31vhS7NQECk7dqwLjkn3aXO2tuKuStgL+XP/7ef9/8vP3w9/YnR+Yc0vcthMjQbGm5VwUi\ntNaRAEqpWUArIGVxbwVM02Zc5Z9KqZxKqXxa69P2DlytQDVmB8+298cKIYRLsWUlpgAgKsXzk8mv\nPeo+Qggh0km6LrOnlOqrlApVSoXGxPzzAqcQQgj7sKW4RwMFUzwvkPzao+6D1nqC1jpIax3k5+f3\nqFmFEELYyJbiHgIEKqWKKqU8gY7A4nv2WQy8rIzqwOW06G8XQghhm1QvqGqtE5RSA4BVmKGQk7XW\ne5VS/ZLfHw8sxwyDjMAMheyRdpGFEEKkxqZx7lrr5ZgCnvK18Skea6C/faMJIYR4XOl6QVUIIUT6\nkOIuhBAuyLL53JVSMcDxx/zjvsA5O8axkhyLY3KVY3GV4wA5ltsKa61THW5oWXF/EkqpUFsmq3cG\nciyOyVWOxVWOA+RYHpV0ywghhAuS4i6EEC7IWYv7BKsD2JEci2NylWNxleMAOZZH4pR97kIIIR7O\nWVvuQgghHsKhi7tSqrFS6qBSKkIpNfQ+7yul1DfJ7+9WSlWyIqctbDiWOkqpy0qpncnbB1bkTI1S\narJS6qxSas8D3nemc5LasTjLOSmolFqnlNqnlNqrlHrzPvs4xXmx8Vic5bxkUUptU0rtSj6W/9xn\nn7Q7L1prh9ww89gcAYoBnsAuoPQ9+zQFVgAKqA5stTr3ExxLHWCp1VltOJbaQCVgzwPed4pzYuOx\nOMs5yQdUSn6cHTjkxP+v2HIsznJeFJAt+bEHsBWonl7nxZFb7ndWgNJaxwO3V4BK6c4KUFrrP4Gc\nSql86R3UBrYci1PQWm8ALjxkF2c5J7Yci1PQWp/WyWsWa62vAvv552I5TnFebDwWp5D83/pa8lOP\n5O3ei5xpdl4cubi70gpQtuZ8LvmfZiuUUmXSJ5rdOcs5sZVTnROlVBGgIqaVmJLTnZeHHAs4yXlR\nSrkrpXYCZ4Fftdbpdl5smhVSpIvtQCGt9TWlVFNgEWbBcWEdpzonSqlswHzgLa31FavzPIlUjsVp\nzovWOhF4VimVE1iolCqrtb7vNR57c+SWu91WgHIAqebUWl+5/U84baZY9lBK+aZfRLtxlnOSKmc6\nJ0opD0wxnKm1XnCfXZzmvKR2LM50Xm7TWl8C1gGN73krzc6LIxd3V1oBKtVjUUrlVUqp5MdVMefm\nfLonfXLOck5S5SznJDnjJGC/1nr0A3ZzivNiy7E40XnxS26xo5TKCrwIHLhntzQ7Lw7bLaNdaAUo\nG48lGHhNKZUAxAIddfLldEeilPoZM1rBVyl1EhiOuVDkVOcEbDoWpzgnQE2gGxCe3L8L8B5QCJzu\nvNhyLM5yXvIBU5VS7phfQHO01kvTq4bJHapCCOGCHLlbRgghxGOS4i6EEC5IirsQQrggKe5CCOGC\npLgLIYQLkuIuhBAuSIq7EEK4ICnuQgjhgv4fYrq/NhCtb38AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d51b1ddc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_new = np.linspace(0, 3, 1000).reshape(-1, 1)\n",
    "y_proba = log_reg.predict_proba(X_new)\n",
    "plt.plot(X_new, y_proba[:, 1], \"g-\", label=\"Iris-Virginica\")\n",
    "plt.plot(X_new, y_proba[:, 0], \"b--\", label=\"Not Iris-Virginica\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.98552764,  0.01447236],\n",
       "       [ 0.98541511,  0.01458489],\n",
       "       [ 0.98530171,  0.01469829],\n",
       "       ..., \n",
       "       [ 0.02620686,  0.97379314],\n",
       "       [ 0.02600703,  0.97399297],\n",
       "       [ 0.02580868,  0.97419132]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5732246,  0.4267754]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg.predict_proba([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multinimoal Logistic Regression or Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = iris[\"data\"][:, (2, 3)] # petal length & petal width"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = iris[\"target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "softmax_reg = LogisticRegression(multi_class=\"multinomial\",solver=\"lbfgs\", C=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=10, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='multinomial',\n",
       "          n_jobs=1, penalty='l2', random_state=None, solver='lbfgs',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_reg.predict([[5, 2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  6.33134077e-07,   5.75276067e-02,   9.42471760e-01]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_reg.predict_proba([[5, 2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class LogisticRegression in module sklearn.linear_model.logistic:\n",
      "\n",
      "class LogisticRegression(sklearn.base.BaseEstimator, sklearn.linear_model.base.LinearClassifierMixin, sklearn.feature_selection.from_model._LearntSelectorMixin, sklearn.linear_model.base.SparseCoefMixin)\n",
      " |  Logistic Regression (aka logit, MaxEnt) classifier.\n",
      " |  \n",
      " |  In the multiclass case, the training algorithm uses the one-vs-rest (OvR)\n",
      " |  scheme if the 'multi_class' option is set to 'ovr', and uses the cross-\n",
      " |  entropy loss if the 'multi_class' option is set to 'multinomial'.\n",
      " |  (Currently the 'multinomial' option is supported only by the 'lbfgs',\n",
      " |  'sag' and 'newton-cg' solvers.)\n",
      " |  \n",
      " |  This class implements regularized logistic regression using the\n",
      " |  'liblinear' library, 'newton-cg', 'sag' and 'lbfgs' solvers. It can handle\n",
      " |  both dense and sparse input. Use C-ordered arrays or CSR matrices\n",
      " |  containing 64-bit floats for optimal performance; any other input format\n",
      " |  will be converted (and copied).\n",
      " |  \n",
      " |  The 'newton-cg', 'sag', and 'lbfgs' solvers support only L2 regularization\n",
      " |  with primal formulation. The 'liblinear' solver supports both L1 and L2\n",
      " |  regularization, with a dual formulation only for the L2 penalty.\n",
      " |  \n",
      " |  Read more in the :ref:`User Guide <logistic_regression>`.\n",
      " |  \n",
      " |  Parameters\n",
      " |  ----------\n",
      " |  penalty : str, 'l1' or 'l2', default: 'l2'\n",
      " |      Used to specify the norm used in the penalization. The 'newton-cg',\n",
      " |      'sag' and 'lbfgs' solvers support only l2 penalties.\n",
      " |  \n",
      " |  dual : bool, default: False\n",
      " |      Dual or primal formulation. Dual formulation is only implemented for\n",
      " |      l2 penalty with liblinear solver. Prefer dual=False when\n",
      " |      n_samples > n_features.\n",
      " |  \n",
      " |  C : float, default: 1.0\n",
      " |      Inverse of regularization strength; must be a positive float.\n",
      " |      Like in support vector machines, smaller values specify stronger\n",
      " |      regularization.\n",
      " |  \n",
      " |  fit_intercept : bool, default: True\n",
      " |      Specifies if a constant (a.k.a. bias or intercept) should be\n",
      " |      added to the decision function.\n",
      " |  \n",
      " |  intercept_scaling : float, default 1.\n",
      " |      Useful only when the solver 'liblinear' is used\n",
      " |      and self.fit_intercept is set to True. In this case, x becomes\n",
      " |      [x, self.intercept_scaling],\n",
      " |      i.e. a \"synthetic\" feature with constant value equal to\n",
      " |      intercept_scaling is appended to the instance vector.\n",
      " |      The intercept becomes ``intercept_scaling * synthetic_feature_weight``.\n",
      " |  \n",
      " |      Note! the synthetic feature weight is subject to l1/l2 regularization\n",
      " |      as all other features.\n",
      " |      To lessen the effect of regularization on synthetic feature weight\n",
      " |      (and therefore on the intercept) intercept_scaling has to be increased.\n",
      " |  \n",
      " |  class_weight : dict or 'balanced', default: None\n",
      " |      Weights associated with classes in the form ``{class_label: weight}``.\n",
      " |      If not given, all classes are supposed to have weight one.\n",
      " |  \n",
      " |      The \"balanced\" mode uses the values of y to automatically adjust\n",
      " |      weights inversely proportional to class frequencies in the input data\n",
      " |      as ``n_samples / (n_classes * np.bincount(y))``.\n",
      " |  \n",
      " |      Note that these weights will be multiplied with sample_weight (passed\n",
      " |      through the fit method) if sample_weight is specified.\n",
      " |  \n",
      " |      .. versionadded:: 0.17\n",
      " |         *class_weight='balanced'* instead of deprecated\n",
      " |         *class_weight='auto'*.\n",
      " |  \n",
      " |  max_iter : int, default: 100\n",
      " |      Useful only for the newton-cg, sag and lbfgs solvers.\n",
      " |      Maximum number of iterations taken for the solvers to converge.\n",
      " |  \n",
      " |  random_state : int seed, RandomState instance, default: None\n",
      " |      The seed of the pseudo random number generator to use when\n",
      " |      shuffling the data. Used only in solvers 'sag' and 'liblinear'.\n",
      " |  \n",
      " |  solver : {'newton-cg', 'lbfgs', 'liblinear', 'sag'}, default: 'liblinear'\n",
      " |      Algorithm to use in the optimization problem.\n",
      " |  \n",
      " |      - For small datasets, 'liblinear' is a good choice, whereas 'sag' is\n",
      " |          faster for large ones.\n",
      " |      - For multiclass problems, only 'newton-cg', 'sag' and 'lbfgs' handle\n",
      " |          multinomial loss; 'liblinear' is limited to one-versus-rest\n",
      " |          schemes.\n",
      " |      - 'newton-cg', 'lbfgs' and 'sag' only handle L2 penalty.\n",
      " |  \n",
      " |      Note that 'sag' fast convergence is only guaranteed on features with\n",
      " |      approximately the same scale. You can preprocess the data with a\n",
      " |      scaler from sklearn.preprocessing.\n",
      " |  \n",
      " |      .. versionadded:: 0.17\n",
      " |         Stochastic Average Gradient descent solver.\n",
      " |  \n",
      " |  tol : float, default: 1e-4\n",
      " |      Tolerance for stopping criteria.\n",
      " |  \n",
      " |  multi_class : str, {'ovr', 'multinomial'}, default: 'ovr'\n",
      " |      Multiclass option can be either 'ovr' or 'multinomial'. If the option\n",
      " |      chosen is 'ovr', then a binary problem is fit for each label. Else\n",
      " |      the loss minimised is the multinomial loss fit across\n",
      " |      the entire probability distribution. Works only for the 'newton-cg',\n",
      " |      'sag' and 'lbfgs' solver.\n",
      " |  \n",
      " |      .. versionadded:: 0.18\n",
      " |         Stochastic Average Gradient descent solver for 'multinomial' case.\n",
      " |  \n",
      " |  verbose : int, default: 0\n",
      " |      For the liblinear and lbfgs solvers set verbose to any positive\n",
      " |      number for verbosity.\n",
      " |  \n",
      " |  warm_start : bool, default: False\n",
      " |      When set to True, reuse the solution of the previous call to fit as\n",
      " |      initialization, otherwise, just erase the previous solution.\n",
      " |      Useless for liblinear solver.\n",
      " |  \n",
      " |      .. versionadded:: 0.17\n",
      " |         *warm_start* to support *lbfgs*, *newton-cg*, *sag* solvers.\n",
      " |  \n",
      " |  n_jobs : int, default: 1\n",
      " |      Number of CPU cores used during the cross-validation loop. If given\n",
      " |      a value of -1, all cores are used.\n",
      " |  \n",
      " |  Attributes\n",
      " |  ----------\n",
      " |  coef_ : array, shape (n_classes, n_features)\n",
      " |      Coefficient of the features in the decision function.\n",
      " |  \n",
      " |  intercept_ : array, shape (n_classes,)\n",
      " |      Intercept (a.k.a. bias) added to the decision function.\n",
      " |      If `fit_intercept` is set to False, the intercept is set to zero.\n",
      " |  \n",
      " |  n_iter_ : array, shape (n_classes,) or (1, )\n",
      " |      Actual number of iterations for all classes. If binary or multinomial,\n",
      " |      it returns only 1 element. For liblinear solver, only the maximum\n",
      " |      number of iteration across all classes is given.\n",
      " |  \n",
      " |  See also\n",
      " |  --------\n",
      " |  SGDClassifier : incrementally trained logistic regression (when given\n",
      " |      the parameter ``loss=\"log\"``).\n",
      " |  sklearn.svm.LinearSVC : learns SVM models using the same algorithm.\n",
      " |  \n",
      " |  Notes\n",
      " |  -----\n",
      " |  The underlying C implementation uses a random number generator to\n",
      " |  select features when fitting the model. It is thus not uncommon,\n",
      " |  to have slightly different results for the same input data. If\n",
      " |  that happens, try with a smaller tol parameter.\n",
      " |  \n",
      " |  Predict output may not match that of standalone liblinear in certain\n",
      " |  cases. See :ref:`differences from liblinear <liblinear_differences>`\n",
      " |  in the narrative documentation.\n",
      " |  \n",
      " |  References\n",
      " |  ----------\n",
      " |  \n",
      " |  LIBLINEAR -- A Library for Large Linear Classification\n",
      " |      http://www.csie.ntu.edu.tw/~cjlin/liblinear/\n",
      " |  \n",
      " |  SAG -- Mark Schmidt, Nicolas Le Roux, and Francis Bach\n",
      " |      Minimizing Finite Sums with the Stochastic Average Gradient\n",
      " |      https://hal.inria.fr/hal-00860051/document\n",
      " |  \n",
      " |  Hsiang-Fu Yu, Fang-Lan Huang, Chih-Jen Lin (2011). Dual coordinate descent\n",
      " |      methods for logistic regression and maximum entropy models.\n",
      " |      Machine Learning 85(1-2):41-75.\n",
      " |      http://www.csie.ntu.edu.tw/~cjlin/papers/maxent_dual.pdf\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      LogisticRegression\n",
      " |      sklearn.base.BaseEstimator\n",
      " |      sklearn.linear_model.base.LinearClassifierMixin\n",
      " |      sklearn.base.ClassifierMixin\n",
      " |      sklearn.feature_selection.from_model._LearntSelectorMixin\n",
      " |      sklearn.base.TransformerMixin\n",
      " |      sklearn.linear_model.base.SparseCoefMixin\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, penalty='l2', dual=False, tol=0.0001, C=1.0, fit_intercept=True, intercept_scaling=1, class_weight=None, random_state=None, solver='liblinear', max_iter=100, multi_class='ovr', verbose=0, warm_start=False, n_jobs=1)\n",
      " |      Initialize self.  See help(type(self)) for accurate signature.\n",
      " |  \n",
      " |  fit(self, X, y, sample_weight=None)\n",
      " |      Fit the model according to the given training data.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : {array-like, sparse matrix}, shape (n_samples, n_features)\n",
      " |          Training vector, where n_samples is the number of samples and\n",
      " |          n_features is the number of features.\n",
      " |      \n",
      " |      y : array-like, shape (n_samples,)\n",
      " |          Target vector relative to X.\n",
      " |      \n",
      " |      sample_weight : array-like, shape (n_samples,) optional\n",
      " |          Array of weights that are assigned to individual samples.\n",
      " |          If not provided, then each sample is given unit weight.\n",
      " |      \n",
      " |          .. versionadded:: 0.17\n",
      " |             *sample_weight* support to LogisticRegression.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      self : object\n",
      " |          Returns self.\n",
      " |  \n",
      " |  predict_log_proba(self, X)\n",
      " |      Log of probability estimates.\n",
      " |      \n",
      " |      The returned estimates for all classes are ordered by the\n",
      " |      label of classes.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : array-like, shape = [n_samples, n_features]\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      T : array-like, shape = [n_samples, n_classes]\n",
      " |          Returns the log-probability of the sample for each class in the\n",
      " |          model, where classes are ordered as they are in ``self.classes_``.\n",
      " |  \n",
      " |  predict_proba(self, X)\n",
      " |      Probability estimates.\n",
      " |      \n",
      " |      The returned estimates for all classes are ordered by the\n",
      " |      label of classes.\n",
      " |      \n",
      " |      For a multi_class problem, if multi_class is set to be \"multinomial\"\n",
      " |      the softmax function is used to find the predicted probability of\n",
      " |      each class.\n",
      " |      Else use a one-vs-rest approach, i.e calculate the probability\n",
      " |      of each class assuming it to be positive using the logistic function.\n",
      " |      and normalize these values across all the classes.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : array-like, shape = [n_samples, n_features]\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      T : array-like, shape = [n_samples, n_classes]\n",
      " |          Returns the probability of the sample for each class in the model,\n",
      " |          where classes are ordered as they are in ``self.classes_``.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.base.BaseEstimator:\n",
      " |  \n",
      " |  __getstate__(self)\n",
      " |  \n",
      " |  __repr__(self)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  __setstate__(self, state)\n",
      " |  \n",
      " |  get_params(self, deep=True)\n",
      " |      Get parameters for this estimator.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      deep : boolean, optional\n",
      " |          If True, will return the parameters for this estimator and\n",
      " |          contained subobjects that are estimators.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      params : mapping of string to any\n",
      " |          Parameter names mapped to their values.\n",
      " |  \n",
      " |  set_params(self, **params)\n",
      " |      Set the parameters of this estimator.\n",
      " |      \n",
      " |      The method works on simple estimators as well as on nested objects\n",
      " |      (such as pipelines). The latter have parameters of the form\n",
      " |      ``<component>__<parameter>`` so that it's possible to update each\n",
      " |      component of a nested object.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      self\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from sklearn.base.BaseEstimator:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.linear_model.base.LinearClassifierMixin:\n",
      " |  \n",
      " |  decision_function(self, X)\n",
      " |      Predict confidence scores for samples.\n",
      " |      \n",
      " |      The confidence score for a sample is the signed distance of that\n",
      " |      sample to the hyperplane.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : {array-like, sparse matrix}, shape = (n_samples, n_features)\n",
      " |          Samples.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      array, shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes)\n",
      " |          Confidence scores per (sample, class) combination. In the binary\n",
      " |          case, confidence score for self.classes_[1] where >0 means this\n",
      " |          class would be predicted.\n",
      " |  \n",
      " |  predict(self, X)\n",
      " |      Predict class labels for samples in X.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : {array-like, sparse matrix}, shape = [n_samples, n_features]\n",
      " |          Samples.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      C : array, shape = [n_samples]\n",
      " |          Predicted class label per sample.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.base.ClassifierMixin:\n",
      " |  \n",
      " |  score(self, X, y, sample_weight=None)\n",
      " |      Returns the mean accuracy on the given test data and labels.\n",
      " |      \n",
      " |      In multi-label classification, this is the subset accuracy\n",
      " |      which is a harsh metric since you require for each sample that\n",
      " |      each label set be correctly predicted.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : array-like, shape = (n_samples, n_features)\n",
      " |          Test samples.\n",
      " |      \n",
      " |      y : array-like, shape = (n_samples) or (n_samples, n_outputs)\n",
      " |          True labels for X.\n",
      " |      \n",
      " |      sample_weight : array-like, shape = [n_samples], optional\n",
      " |          Sample weights.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      score : float\n",
      " |          Mean accuracy of self.predict(X) wrt. y.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.feature_selection.from_model._LearntSelectorMixin:\n",
      " |  \n",
      " |  transform(*args, **kwargs)\n",
      " |      DEPRECATED: Support to use estimators as feature selectors will be removed in version 0.19. Use SelectFromModel instead.\n",
      " |      \n",
      " |      Reduce X to its most important features.\n",
      " |      \n",
      " |              Uses ``coef_`` or ``feature_importances_`` to determine the most\n",
      " |              important features.  For models with a ``coef_`` for each class, the\n",
      " |              absolute sum over the classes is used.\n",
      " |      \n",
      " |              Parameters\n",
      " |              ----------\n",
      " |              X : array or scipy sparse matrix of shape [n_samples, n_features]\n",
      " |                  The input samples.\n",
      " |      \n",
      " |              threshold : string, float or None, optional (default=None)\n",
      " |                  The threshold value to use for feature selection. Features whose\n",
      " |                  importance is greater or equal are kept while the others are\n",
      " |                  discarded. If \"median\" (resp. \"mean\"), then the threshold value is\n",
      " |                  the median (resp. the mean) of the feature importances. A scaling\n",
      " |                  factor (e.g., \"1.25*mean\") may also be used. If None and if\n",
      " |                  available, the object attribute ``threshold`` is used. Otherwise,\n",
      " |                  \"mean\" is used by default.\n",
      " |      \n",
      " |              Returns\n",
      " |              -------\n",
      " |              X_r : array of shape [n_samples, n_selected_features]\n",
      " |                  The input samples with only the selected features.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.base.TransformerMixin:\n",
      " |  \n",
      " |  fit_transform(self, X, y=None, **fit_params)\n",
      " |      Fit to data, then transform it.\n",
      " |      \n",
      " |      Fits transformer to X and y with optional parameters fit_params\n",
      " |      and returns a transformed version of X.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : numpy array of shape [n_samples, n_features]\n",
      " |          Training set.\n",
      " |      \n",
      " |      y : numpy array of shape [n_samples]\n",
      " |          Target values.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      X_new : numpy array of shape [n_samples, n_features_new]\n",
      " |          Transformed array.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.linear_model.base.SparseCoefMixin:\n",
      " |  \n",
      " |  densify(self)\n",
      " |      Convert coefficient matrix to dense array format.\n",
      " |      \n",
      " |      Converts the ``coef_`` member (back) to a numpy.ndarray. This is the\n",
      " |      default format of ``coef_`` and is required for fitting, so calling\n",
      " |      this method is only required on models that have previously been\n",
      " |      sparsified; otherwise, it is a no-op.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      self: estimator\n",
      " |  \n",
      " |  sparsify(self)\n",
      " |      Convert coefficient matrix to sparse format.\n",
      " |      \n",
      " |      Converts the ``coef_`` member to a scipy.sparse matrix, which for\n",
      " |      L1-regularized models can be much more memory- and storage-efficient\n",
      " |      than the usual numpy.ndarray representation.\n",
      " |      \n",
      " |      The ``intercept_`` member is not converted.\n",
      " |      \n",
      " |      Notes\n",
      " |      -----\n",
      " |      For non-sparse models, i.e. when there are not many zeros in ``coef_``,\n",
      " |      this may actually *increase* memory usage, so use this method with\n",
      " |      care. A rule of thumb is that the number of zero elements, which can\n",
      " |      be computed with ``(coef_ == 0).sum()``, must be more than 50% for this\n",
      " |      to provide significant benefits.\n",
      " |      \n",
      " |      After calling this method, further fitting with the partial_fit\n",
      " |      method (if any) will not work until you call densify.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      self: estimator\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(LogisticRegression)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150,)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], shape=(0, 1, 2), dtype=float64)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:0][:, np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1d51b8e2400>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4XHX1+PH3ubNnaZY23fdSCoUCLaHsUFZZyqIssgt+\nFcHlqz9UVET5uuMuCgoIqCiorIIIIqvspS1bKWUpULrQfcnSZDLLPb8/7jTJZCbJpJlkksl5PU8f\nMp/53HvPhCdnZu4993xEVTHGGFNcnEIHYIwxJv8suRtjTBGy5G6MMUXIkrsxxhQhS+7GGFOELLkb\nY0wRsuRujDFFyJK7McYUIUvuxhhThPyFOvCIESN08uTJhTq8McYMSosXL96kqjXdzStYcp88eTKL\nFi0q1OGNMWZQEpEPcplnp2WMMaYIWXI3xpgiZMndGGOKkCV3Y4wpQpbcjTGmCFlyN8b0O9U4mlyD\navPObZ/cjCY3po+5dWhyHbYAkafbUkgRmQDcCowCFLhRVa/pMGcecB/wfmroHlX9bn5DNcYUA3f7\nX6Dxl0AC1EUjpyPDrkAk0O22mvgA3XYZJN7yHvsnQ/m3Yfv1EFsAOOBUQ8XVSOjAPn0dA10ude4J\n4Muq+pKIlAOLReQRVX2jw7ynVXV+/kM0xhQLjT4EjT+F9p/Ym+9GxYcMu7LrbbUF3XIWuFsB1xtM\nvA1bL8A7CZHwxty16NZLYMR9iH9yH7yKwaHb0zKqulZVX0r93AAsA8b1dWDGmOKjjdelJ3YAotB0\nB6qxrjeOPgYapTWxt3JpTeyt4mjTn3sV62DXo3PuIjIZmA0syPL0QSLymog8JCJ7dLL9xSKySEQW\nbdy4MdsUY0wxS67v5AkFt6Hrbd21oC05HigBiZxu5CxaOSd3ESkD7ga+pKr1HZ5+CZioqnsBvwH+\nkW0fqnqjqtaqam1NTbetEYwxxSYwK/u4lIBT1f22EszxQGEIzu1RaMUmp+Qu3pWOu4HbVPWejs+r\nar2qNqZ+fhAIiMiIvEZqjBn0pPwrQASQdqNhKP86It2ko8B+4J8JhNoNBkEqUvvcwQ9OOVLy8TxF\nPTh1m9xFRICbgWWq+otO5oxOzUNE5qb2uzmfgRpjBj8JzESG/w2C88CpgcDeSNVvcEo+2v22Ikj1\nH6DsM+CMB2cMlF4EIx6H8q+Bbyo4IyFyBjL8H4hT0fcvaACT7mpCReQQ4GlgCW1XMq4AJgKo6vUi\n8nngUryrGs3AZar6XFf7ra2tVesKaYwxPSMii1W1trt53ZZCquozpH+HyjbnWuDa3MMzxhSSxpeg\n9d+D+BKQMig5Dyn7HCL57QLubrsCovfgfS70Q+RCnIrL83oMk13B+rkbYwpDE++jW84HbUoN1MH2\nm9HkWqTy6rwdx932NYje224kAc034Tp+nPLL8nYck521HzBmiNHtN2UpKYxC9AE0uSkvx3BdF6JZ\ni+Zg+815OYbpmiV3Y4aa+FIgmTkuIUjmqza8Ca9bSdYA8nQM0xVL7sYMNYHdAV/muLaAb2KeDlJC\n55fquu8hY3rPkrsxQ4yUfgroeDNQGMLHI7783FzoOA6EOmk1VXJhXo5humbJ3ZghRvzTkOpbwb8n\nIKlqmQuQih/m9ThO1c8hfCptacYHkU/gDPtqXo9jsrNqGWOGIAnujYy4B1Uldf9hn3AqfwL8BNd1\nvU/zpt9YcjdmCNLkWrTxN9DyDOpUI6WfQoOHQtMN0PwQSBhKzobIGdD0J2i+G3AhfAqUfBKiD0Lz\nn8DdDqGjkLLPIr7hnR6vfWLX+Dto4zUQfxV8E7z6+tDBmTG69ej261PxhKDkHKTkXETSrxeoumjz\nXV6cOcZTCBp91Hs9yQ0QnIuUfQHxT+qz43V7h2pfsTtUjSkMTW5AN80HbaCtaibsJVBtBna03o2A\nRFL18NHUWMhr8qVRvJvRAQLgVCMjHkSc8q6PHV+Gbjk7dZwduScMw76PU3Jy2zyNoptOguTa9HhC\nh+NU/Tptn27dt6H5vnbx+MEZjoz4F+IM68Fvpu+422+Fhp/TFqMDUuK1SfD37CJ2rneo2vckY4YY\n3X4L6HbSyyGj3s1MtO+p3gy6hbbEDtACupW2JAUQB7cObbqj+2M3/Dz1ZtH+Q2UUGn+Ears+7c0P\neJ9wO8bT8iSaWN62v+Q6aL6nQzyJVDx3dhtPf1BtgcZfkB6jC9qEbv9tnx3XkrsxQ01sAfmvNY9C\n7Pnup8Vfyz7uNqRWWPJo7EXSk2GKOBBrt4/46520Ac4xnv6QXEX2slAXYgv77LCW3I0Zanzj6aZd\n1M7sFHw5nD/utNRSwClrN28CmeWaqXm+Ue3mjQbNckMWvjzW7PeSMxy0kzdT39i+O2yf7dkYMyB5\nde6hDqN+MtOBk2Vsx3jHWowgUnpuDse+lPTe6wBhiJyOSFtMUnIGSMcbrRxvQY9gu4Wv/XuAf1KW\neAJI6XndxtMfxKmC0JFk/s4jSOklfXZcS+7GDDES3BsqfgxShZdogxA6DCpvBGccEPbGAntD1Z/A\nvxteYgp5PdOrboFgrTeHMDijkKrrEP/U7o8dmQ/lXwIp9S7MEoLIyciwb6TP841Gqm7JiEeqb0tb\n1ENEkKo/ZInntznF01+k8scQPhovxgjIMBj27axVQnk7plXLGDM0qSYhuQacYYhTmRpTcD8EQoiv\nbTE1Ta4HXMQ3pm3M3eKVHvrG97hWXjUGyXVelU370zEZ87LHk3VuL+LpL+rWg7sNfGPwFrjrubz1\nczfGFCcRH3QowxMR8I3LnNv+PPeOMacanOouj6GJlej2P0HiLQjuhZRc4H0ql2DasdVt9KptWp4A\n30hvXnDvTuPJ+npyiCffVBVaHkGb7gYSSOSjXhuHjFNKO2IcBv1UnmnJ3RjTJzT+GrrlAtAYkID4\ny2jT32H4HYh/Wts8txHdfGqq9DEKcUGjj6DDrsIpOa1g8edC66/wbuhSr7JH44u8x5XXFfzbg51z\nN8b0Ca37dqqmPZEaiYM2ovU/Sp/X9GdIrqetnl69nxu+h2qUgUrjb0Lzv1oTuzfYDLHnIF74U86W\n3I0xeacag8Sb2Z5J1dm3E30U6Lh4CIAD8Tf6ILo8iT1H1r742oS2PNXv4XRkyd0Y0wf8dNq3XUrT\nHztV2edpEpyKvEaVV1JB9tcY9MofC8ySuzEm70QciJxKZm13GErOSZ9begGZte8O+CemnZsfcMLH\ndnIvmAPhTnrZ9yNL7saYPiHDroDg/njNxsqBIISPQsouTZ8XOgzKPpuaV+bVv/umIFU3FCLsnIlT\njlT93vsEL2Wt/6TqN4hvZKHDszp3Y0zf0sQH3tqs/l2QLm63V7cO4ku80zT+mQWvNsmVahzir3in\nkYJzvDLPPmR17saYPqOJVWjzfaD1SGie1xLA3YI2/wPctUhwPwgdBRpFW56DxHIksAmNnAA4EH0Y\njb0C/slI5GTEqUCcCggd0nYMVYgvRqOPgkS8ef4phXvNncQjEoDgfgWLqzP2yd0Y0yNu80NQ9zW8\nSpE4UAKBmRBfCrhAi3dqxRkH7qZ2vd9LUs3Bgl4rYW3C6yMfRKpvRwK7th5DVdH6r0Pzv/FKJH3e\nv2FX4pR8vJ9f8Y54vuEtHFLgeKyfuzEm79Rtgrqv4yW4HZ0OmyC+GC+Bp0oatQmSy0G30da6twnc\njeCuSSV2vP1oPVp3efqBYs9D9N+pbRWvVr4F6r+PtmsN3G9iz0P0oYETTw4suRtjchd/MUu3Rkhf\nfKP9WMfxbGNA4h3vnPuOWe3u+kzng5Zncg43XzT6UBfxPN3v8eTCkrsxpgd2rtlV9xTvVEf742RJ\nTyIghbhU2FU8ffU76R1L7saY3AX3I3vayFbZkq0ffLYxHwTmpHWHlMipZF+sw4XgYbnHmydSMrDi\nyYUld2NMzkSCSNXvvAumUoLXaz0E4VOBqtTdpyEgAsGDvdWQdoxJCfhngH+ftl7uUgrOSKTyp+nH\nCe4NZZ9J7SsMlAARpPLXiNPhDtd+IIG9oOySDvGEkcprChJPLqxaxhjTY+o2QsvjoA0QPBjxT/b6\nybQ84XV3DM5BAnt4i17HnoXE+15iD871dhB/2auu8Y2H0KFIJ6daNLkGWp4CCUHoaK9lbgENhHhy\nrZbpNrmLyATgVmAU3omxG1X1mg5zBLgGOAFoAi5U1Ze62q8ld2Pyz0umL6RuGtoVAnO8JxJL2pJp\n8CBEfGjiXW+BZqcKQkcgEkST67wLhBL2xrpYSKNYqGq73884782qk37sWbd3t3lvaupC6PBuFxXp\nrXzexJQAvqyqL4lIObBYRB5R1fbt2o4Hpqf+7Q/8LvVfY0w/UXcLuvk8b+UidUEc8E0DiXh3fqJe\npYtUocG9IPoYIKnqlwAaPgma7wAcb1tcry95uxuLio1qDN36aYi9Quvvx6mC6tsR3+hut/dq/i8H\nfCAK9S5aoFr8jrpN7qq6Flib+rlBRJYB44D2yf0U4Fb1vga8ICKVIjImta0xph9o3VWQXEFr/3QF\nEm+kfnDbxrQJoh+mjwE0/7ndzlL/2fZ5qHkOcUr6NvgC0cYbIPYSbfX5QDKK1l2OVN/a9bbJzanE\n3m5b8GrfgwciHVa56m89uqAqIpOB2UCHhsyMA1a1e7w6NWaM6QeqSWh5jLaFMXZI0prE22ZnGeuM\nQKzwvcn7TPOdZPaST0JssXddoSst/yF7Ck16dfoFlnNyF5Ey4G7gS6pavzMHE5GLRWSRiCzauHHj\nzuzCGJOVS+4JuweU1DJ5xSq+k88BGifrYh24oNkWH+lfOSV38Zbpvhu4TVXvyTJlDTCh3ePxqbE0\nqnqjqtaqam1NTc3OxGuMyUIkAIF9yaw3lyxjdDKWTTytmVfRCR1H1huz/FO7X3AjNI/sv8cgEj66\n97H1UrfJPVUJczOwTFV/0cm0+4ELxHMAUGfn243pX1Lxg9TqQDsWvigBZzg4o1N15eA16irzKmla\nxwJACAL7exdfEby7RcNQ/g3Eqe7fF9KPpPx/wZf5+5GKn3S/rX8ilF3qbYOD93uLQMnHkcAefRd0\njnKpljkYOB9YIiKvpMauACYCqOr1wIN4ZZDL8UohL8p/qMaYroh/MtQ86rXdTSwH/55IZL5XARJ9\n0Gux65uElHzUWzyj5Qm05VlwRiAlp3lvArEFaPQ/4JQikVMQ/y6Ffll9SpwqGPFgu9/PRKTkozm/\noTlln0VD89DmfwJJJHw8Epzdt0HnyG5iMmYI0OQmSLwNvnGIfxIAbuJDr9OhbzyEjsFxsn+RV01C\n/DUgCYG9vVNAA4xXq/4GuI0QmFW01T1gi3UYY/BuatL670LzXd5dlRpDg3NAfRBv310xhFt9O05w\nVvr2sZfQrZ/FqygRwIHKa5DQwf34KrqmiRXo1k95veNxQJPosG/hlJxe6NAKynrLGFPEtOl2aL4X\niHmtAmiB2IsdEjve+Jb0havVbUS3/k9qYY3toI1e7/Vtn/W+CQwAqi665UJIrvLq97URaIb676Lx\nJYUOr6AsuRtTzJr+SNtiGTtkK98DaMGN/rftYfRhyHbaVl2IPpCf+Hor/jJoHZk94mPeG9sQZsnd\nmGLm9vCWlOSHbT/rNrLXereg7pbeRJU/7jaylyO6MEC+XRSKJXdjilnoYHr0Zx45vu3n4IGkL6CR\nIhEkOEDOuQdnp24m6igCocLXmheSJXdjipiUXebVtbcuNOHD60me5U8/fDKOU9m2bWAmhD9CW908\n3s/B/dta9xaYONVQ9lnSYwyDfyJSckqhwhoQrFrGmCIm/gkw4kG06VaILQL/FKTkk6iUQt23IPGy\nl/xLL8EpPSdz+4ofQ+gItPkuIIlEPgrhk/DubRwYnLJL0cBeaNNt3mma8HFIyRmIhAsdWkFZnbsx\ng1hjdDXbm1cworwWn99LZq67DRLvgn8GThf92FVdcNeBlCNOeV7icRPrvZbD/j1wnGDqOFFwN4NT\ng0i2pep6Ho8mNwMu4ut5G5Nc4xmorM7dmCIWjW2kfu1JDA9uIQJoFJa3HMvUsnWQeK11nhs8BCpv\nyrhByW1+BBqu8m76wUVDRyAVP9rpxTlcdxts+hi4q1Mjghs+A5wKaPqzt5A0gpZ+Bim9JOOTv0Yf\nQevbxzMPqbg6Ix5NrEC3XebdkAWofzJS8XMkMKPbGFVdtOHn0PSX1DVYBy29OGs8xcA+uRszCG36\nYC5VwW20z0k7/pQz8lRoPk5VW1sojb/mLepBtN2kIATn4lTfslPxuBsOBzdbOykf6aWXESj/Ok7p\n2e3iWYJuPrfbeFRb0I3zwN1KWgdMGYbUPNHtp3238Vpo/D3ppaGZ8Qx0uX5ytwuqxgwydY1vZiR2\n8JJ61g+gLQ+lPdTGm8jsYR6D2EJvjdAechMrOknskFlT3wzbb9i5eKKPeV9ROrY21jhE/9VljKoK\n228hs+a/GbZf3+W2g5Uld2MGmYbm13u4RYcEm1xJ5k0/gAQgua7nAcWX92y+26H+vNN4gunxuGs7\n6ZPenMObUty7yzZrPJu72XZwsuRuzCAzouLwrOOdnmGVSPrj4Fyy9jDXOPin9zygULdnCNIFdusQ\nz36dxBNLjycwy0v4HUkJEti7y0OKBL3Fr7Pxd3++fjCy5G7MIBMO1rCyeXZaMt/xc9b8XvbVtIdS\n+slUwm//5x+B0osQZ1iP43GcSgge0cmzHZN2GCn/Wod4/ifVT71DPCUXpscT2A/8M/Hq9HcIgm9i\nauGMrkn5lXi91zvEM+zr3W47GFlyN2YQmjL176yInUpL0k9Sha3xCtYHfoWUXZG6aUlAKmHYD3FK\nz0vbVnyjkeH/gPDJ4IwE/65IxXeQsv+30/E41TdAyUV4yVO83vBVNyNVt0CgFpwaCB6MVP8ZCe7X\nIZ5RyPB7M+Mpvyx9nghS/Qco+ww448EZ670hVf8Vke4L/yR8JFL1e+9Noot4ioVVyxhjzCBide7G\nDHCaWOn1Wo89513MDJ+MlH8dcUrzepz/rnifHzzzJO9t3cqISAmfm7s/583apyhru00bS+7GFIC6\ndejmM1Ltal3QBDTfiybeguq/5y3xPr9qJZc+eD/RRAKADU3bufqZp2iKxflM7cDoD2P6hp1zN6YA\ntOlu0GbSa7ZjEH8rtaRdfvz8+WdbE/sOzYkE1y1aQMJ1O9nKFANL7sYUQmIp6Xdkpoh4fWHy5N2t\n2Wu448kkW6Mdb+gxxcSSuzGF4J9JZlnejuem5e0wUyqrs44HHB+VoaHdNbHYWXI3pgCk5DRvweq0\nP8Eg+HaBwF55O86XDzqYsD/90lrE7+eS2rkEfFkW4jBFw5K7MQUgTiUy/M7UakcOEILISUj1H/Na\nxXLwhEn85vj5TKmsAmB4pISvHHgIl9rF1KJn1TLGFIj4JyPVf0BV+7Qs8agp0zhqyrQ+P44ZWCy5\nG1MgddEov124gAeXv03Y7+fcWXtz/l774OvYe12VO5Yu4Y+vvkRjLMbRU6bx+bkHMqKkpEfHa5/Y\n1zTU8+sFz/PMyg+ojkT49JxaTtp1t4Ilf02uRRt/Ay3PgFPttSQIz7c3o16wO1SNKYBoIs7xt93K\n2sYGYkmva2PE7+eIKVO59viT0uZ+8/FH+Mebb9CcKmkMOA7VkRIePu9ChoVCGfvuzobtjRx3259o\naGkhmfr7j/j9fGrOfvy/Aw7q5SvrOU1uQDfNB22gtYOlRKDkkzjlX+z3eAY66+duzAB2/1tvsmH7\n9tbEDl79+ePvv8fyLW3lix821HPPsqWtiR0g7rrUtUS5Y+mSnTr2719axPZYrDWx7zj2jYtfpL4l\nW0vdvqXbb0m1423XmlibYftNqNvQ7/EUC0vuxhTAC6tX0ZyIZ4w7Iry6vq2H+dINGwg4mVUt0USC\n51at3KljL1i9iniWG5iCPh9vb96UZYs+FlsAZP4ukEDrcnqm5yy5G1MAEysqCWZJ2oIwuqxt3dBR\nZWW4WU6d+kSYWFGxU8ceP6yCbGeyY0k37dj9xjceskWkcXBG9Xs4xcKSuzEF8PE9ZuHvcOHUJ0J1\nJMKB4ye2js0aOYqJlZX4O1xYDPh8XLD37J069sX77keoQ+17wPFRO3Ys44ft3BtGb0jpp0jv0Q4Q\ngOAcxD++3+MpFpbcjSmAMeXl/OHUjzF+2DDCPj9Bn4+9R4/hr6edidMukYsIt556OvuNG0/Q5yPs\n9zOqtIwb5p/C1Krsd592Z5/RY/j5McdRHY4Q8XvHPnzyZK474eR8vbwekeDeUPETkCogAgQhdBhS\neW1B4ikWVi1jTAGpKh82NBD0+6gp6brV75bmJrbH4owfNiwvJYJJ12VNQz0VoTAV4cK3IlBNQnIN\nOBWI0//fIAaLvPVzF5FbgPnABlXdM8vz84D7gPdTQ/eo6nd7Fq4xg9PKum3c8vJi3tq8ib1HjebC\nfeYwuqw8p23X1NVx6h23sbnZa+C1+4gRPHD2BTz87nLueuN1Eurysd1mcuL0GSz8cA1/ee0VtrVE\nOX7adE6fuWfGqRWAF1av5Dv/fYIPG+qZUlXF9+YdzZ4jAmjTnyG2CPxTkdJPgDMKafo74xNPgo5E\nnQu8T9AFJOID/8TuJ5qcdPvJXUQOAxqBW7tI7l9R1fk9ObB9cjeD3avr1nLuvXcSSyZJuC4Bx0fY\n7+OeM89hWvXwLrfd1NTE3Jt+l/W5En+AplQlTYk/wLjyclY31LeWQ0b8fqZWVXPXGWenJfg7ly7h\na4/9J21fY0saeOzEfxKQFiAG+AA/OBXg1uN1phQgBMOuwik5bed+Gabf5K3OXVWfArbkJSpjisg3\nn3iUpni8tS963E3SGIvx/aef7Hbb0++4rdPnmtqVSDYl4ryzdUtanXtzIsF7W7dw31vL0ra76snH\nM/b15Vkv4mgjXmIHr5a8BdwNtLUcVu/nhu+hmqUNsRmU8nVB9SAReU1EHhKRPfK0T2MGrFgyyZub\nNmaMK/DC6tXdbr+yvr5Xx29OJHj43XdaHzfGYkSTiYx5h45ejc/J9bqaA/Fl3U8zg0I+kvtLwERV\n3Qv4DfCPziaKyMUiskhEFm3cmPmHYcxg4XccAk72P5+yYKDb7X1ZK81z54gwot0F2I5tfXdoiHcf\nSytNgjOsV3GZgaPXyV1V61W1MfXzg0BAREZ0MvdGVa1V1dqampreHtqYgnFE+NjuMwl16InuNQDb\np9vtz9mzZz3bO74VBH0+zpnVdgHU7zhMq6rK2O5Pb88imuyY4P14597bc8A/EcnjQiGmsHqd3EVk\ntKTqskRkbmqf2df2MqaIXHnoERwwfgIhn5/yYJCQz8cxU6fxuf3273bb7xx5NKOzlD4eNG4ClaEw\nZcEgZcEg5cEgPzziGMaWD6M0EKA8GCTs93PlofPYe9TotG3/ftpZVIUjaWOPrT8Qf+QUIAhSDoQh\nMBtKLwFCIGUgJeCbglTd2IvfhhlocqmW+SswDxgBrAeuAgIAqnq9iHweuBRIAM3AZar6XHcHtmoZ\nUyxWbNvKB9u2scvw4Ywr79lpjTc2rOeKxx9hWCjE9fNPpSQQIJ5M8vK6tSRdl33HjiPo86GqvLp+\nHQ2xFmaPHktZMNjpPp9d+QEvr/uQgyZMYs6YsQBocr3Xp8U3tvXTubp1EF8CThX4Z1p73UEi12oZ\nu4nJDHnLNm3kgbffJOm6nDB9Bnt1+EScD7FEgmsXLeCx995leKSErx50CLM6OU62eF5cvZpfLXiO\nhlgLJ06fwcVzatkcbebeZW+wrrGB/caN55ipu2S0NCg0dRvR5n9CYjkS2AMixyMS6X5D0ylL7sbk\n4LqFL3DdwgVe611VQn4/5+81m68fcljejtEYi3HQLTfQGIuljV9+0KFc0mG5u2zxTKuq5vWNG9Lm\nVYZCxFyXpOvSkkxSEggwpbKKO04/i0igBxdR+5AmVqKbzwCN4n2pLwFnGDL8LsQ3stDhDVrWz92Y\nbnywbRvXvvgC0UQCVxUXr8Tw1tdeZlmHZNob33js4YzEDvDT554m2q5+fWVd9ng6JnaAbS0tNMXj\ntKT6wTfF4yzfsplbXl6ct7h7S+uvBK3DS+wATeBuRBuuLmRYQ4YldzNkPb7iPbJ9b40lkzzy3vK8\nHeeJFe9nHVfggbffan382Pvv9eo4Lckk/3hrYNSpqyYgthDo2Dc+CS2PFSKkIceSuxmy/I6T1oFx\nB58IAV9mr/Wd5eviQmXY33Ycv+P0+qJmPuPuHSFrj3bAlm7uH5bczZD1kWm7kO2akyPCCbvMyNtx\nTp6xe9Zx7zi7tj4+btp0enMJLOL3c3YP6+f7iogPQkeSmcgDED4p2yYmzyy5myFrZGkZVx/9EUI+\nHyX+ABG/n5DPx5WHzmNSZWXejnPV4UcyIcsiGL/6yAk47apbakpLufroYwn5fETaxdP+DWCH6VXV\nDI+UUBoIpub7OWTi5AGT3AGk4rvgmwBSildTXwL+XZHyrxQ6tCHBqmXMkLe5qYlH338XV5Wjpkxl\nZGnfLDX38Lvv8M+33mRUaRlf2P8AKsPZSwKzxbNxeyO/fvEFtkWjnLXHLA6eOIlYMsnj77/Hhu2N\nzBkzlj1HDrwl6VRdiD0LiffBPwOCc62evpfy1s/dmIFOVXlt/Tpe37iB8eXDOGTiJHw9qPceXlLC\nx/eYlTa2qq6Obz/xKHUtUT65z77Mn7EbTbEYv120gNX19Rw7bTonTN+VhOtyy8uLWLpxI7Vjx3Lu\nnl5LgLuXLeWZVSuZXl3NxXP2I+j3s0tVNQdNmEh1pISI3ytX/LChnqc/WEE4EODIyVMpD4VwRAg4\nDklVHPFeR01pGd874ui0GIM+H8ftMr03v7o+J+JA6FDvn+lX9sndDGotiQQX3X8Pr65bh6ricxyq\nIxHuOP0sRu3kYs8/ePpJbu5QUlgRDFIfi6VV1wyPlFAXbSbR7m8o7PMR8PloaFf66BPhiMlTeWbl\nByDgE4eAz+GUXXfnb0tfwxHBEUEVPrHPbG55+SV8joBCUl2uOvxIzhpAp1tMYdlNTGZI+MXzz/L7\nlxbR0q7drU+E/cdP4C8fPaPH+9va3My+v/9tPkPslEDWUsyOQj4/D5/3CSZW5O86gBm87CYmMyTc\n+cbraYl6W/uiAAAZ/ElEQVQdIKnKi2tWZ71xqDtXP/vffIXWrVw/Vrnq8q929fDG5MKSuxnU4m4y\n67jgLQDdU9FE9v0VUtLV1jtRjcmVJXczqB03bdeszbJ2qR5ORTjc4/1ddsBB+Qgrr4J+r5WwMT1h\nyd0Mal8+8GDGlJVTkmqWFfb7KQ+G+Nmxx+/U/iZVVnHI+IkZ49lWTsq2EpNA1rtep1RWtcYYdHyE\n/X4OGDeeEn8AwbtOEPb7OXLyVMJ+Pw7eRdaI3885e+7FHgOwzNEMbHZB1Qx60UScf73zNi+vXcuU\nqio+tttMqiK9ayt7zxtL+dnzzxBNxDlq6jR+cOSxLN+yiaufeZr1jY0cMnESXz7wYLZEm/nh00/y\nzpYt7D1qFF8/ZB5+R/jxs0+z+MMPmVxZyTcOOZwJFRU8/v67PL3yA0aWlnLa7nswpqycF1av4j/v\nLSfiD/DR3WYyffhwXt+wnn++9SZJdTlx+gxmp3qyGwNWLWNMzpKuyyvr15J0ldmjxxBILY7x+sYN\nNLa0sPfoMa2furNZXV/Him3b2KW6mtFl5Z3Oq29pYcmGdQyPlDBj+Ige3czTk3hMcbObmIzJwaIP\n13DJA/fRkkwi4p0eueLQeVy38AU2NTXhEyHhuvzf4UdyRocbnaKJOF946F88s3IFQZ+PlmSSE6fP\n4MdHfyTjOsCNixfyyxeeI+hzSLjKxIoK/nDKx7p8M9jh/W1buei+u7uNx5j27Jy7GbLqW1q46L67\n2RJtZns8RmMsRl1LC1979GFW1dXRFI/TEIvRnEhw1X8f5/UN69O2/8FT/+WZlStoSSZpiMWIJZM8\ntPxtfrdwQdq8pz9YwTULnqMlmUjtz+u9/qn77+02RleV8++5M2s8SzrEY0x7ltzNkPXwu+902oWx\n43AsmeQvr73S+thV5a5lr2eUKEYTCW5tNw/gllcW05zIrMV/b9tW3tu6pcsYF69dQ11LNGs8t3U4\njjHtWXI3Q1ZdNNppnXxHriobm7a3Pk64LvFO6ug73jy1pbk56zy/47AtGu02RslSqdMxHmM6suRu\nhqwDx0/IucFYxO/n2Km7tD4O+nzMGD4iY54Ac8eNTxs7aso0QlkW0Ui6ysyami6PO2fM2KxvQB3j\nMaYjS+5myNpj5CiOmzY9rfIk4vcWmo7422oNwn4/kyurOHW3mWnbf/+Io4n4/a0rLQUch9JgkCsP\nnZc278J9ZlNTWkrI5+1T8JLztw6bR9jfddVLdaSEz889ICOeSZVVnLJb9kVAjAErhTRDnKvKv5e/\nzd+XLiHpKqfP3IOTdt2N51ev4s+vvcK2aDMnTJ/BmTP3JJKl/PD9bVu5+aVFvLl5E/uMGs0nZ+/L\n2PJhGfPqW1r465JXeWzFe4wsLeWifeaw75hxOcf5zMoPcorHFD+rczf9zlVlbWMD5cEQw0KhQoeT\nlaqyfnsjIZ8/7UanTU1NqCo1paUFjM6Y7lmdu+lX/17+Nt964jG2x2O4qhw5eSo/OeY4yoLBQofW\n6qW1H3LZfx5kfWMjrsLs0WO47KCD+f5/n+DtLZsBmFxRya+OO5HdRnR9LtyYgc4+uZtee3XdWs6+\n5w6i7cr9gj4fB4ybwB9PPa2AkbVZ19jA0X/+A03xeOuYTwTF+8bR3rBQiKcv/DTlA/TbhxnarJ+7\n6Tc3LF5IS4c67lgyyYI1q1jTUF+gqNL9fekSEh1KF5OqGYkdIJ5M8q93rH+6GdwsuZteW1lfl3Xh\niaDPx/rGxn6PJ5sV27YRy7EnenMiMWDelIzZWZbcTa/tP3Z81va3sWSSXaqrCxBRpv3HjW9dlLo7\npYEAe48a3ccRGdO3LLmbXvv0vrWUBAJpfcwjfj+fmlPLsFDPF8zoC6fM2J3hJZG0N6Gw309lONxa\nfw7et41JFZUcMXlqIcI0Jm/sgqrJi9X1dfzi+Wd5btVKqiMRLt53P06ZsXuP2tr2ta3NzVz74gs8\n9O7bhH1+zpm1N2fvuRc3v7yYu5ctJakup87YnUtr96d0AFX5GNOe1bkbY0wRyludu4jcAswHNqjq\nnlmeF+Aa4ASgCbhQVV/qechmqNoWbebce+5k2aaNgFeK+Itjj+fIKZnrhn790Ye5a9lSXFX8jsMn\n95nD1w85PGPea+vX8Z3/Ps5r69dRFgxywd6z+cLcA7Out9pRSyLBj599mrveeJ3mRJy548Zz1eFH\n8uh773Lzy4upa4my+4garjr8SGrH5n6XqTH9qdtP7iJyGNAI3NpJcj8B+AJect8fuEZV9+/uwPbJ\n3eyw743XsTVLd8QHzj6PmTVta4f+v4cf5L63lmXM+2ztXL5y0KGtj9/buoWT//aXtJr2sN/P/Okz\n+Mkxx3Ubz0X33c0Lq1e1tvMVvA6OPsdJq+UP+/3cefpZtr6p6Vd5q3NX1aeArppOn4KX+FVVXwAq\nRWRM7qGaoezx99/NmtgB/u/Jx1t/dl2X+7MkdoDfv5T+ISFb3X00keD+t99kU1NTl/G8u2UzC9as\nTuvTrkDcddMSO3if8H/z4gtd7s+YQslHtcw4YFW7x6tTY8Z0a+GHazp97r2tW1t/bozFstbSAxl9\n1Zdu3EAyyzfSkM/Him1bM8bbW751S06nbsBL+m9u3pjTXGP6W7+WQorIxSKySEQWbdxofxQG9uvi\nnPWUyqrWn8uCwSxLVng61tjPHFHT2oa3vVgyyaTKyi7jmVZVnXEna2cEsvZ0N2YgyEdyXwNMaPd4\nfGosg6reqKq1qlpb080iBWZoOHLKNCo7qYW/at6RrT87jsOJ02dknfc/s/dNe/yZffcj2GFxjLDf\nzwnTZ1BT0nXXx12qh1M7ZlzG4hoBxyHsS68/CPv9fGHugV3uz5hCyUdyvx+4QDwHAHWqujYP+zVD\nxKPnX8Su7T4BlweDXH/iyezZ4ULlr4+fz6kzdm+9Wconwif3mcPlBx+WNm9a9XBu+9iZzBo5CsH7\n1H/h3nO4+qhjc4rnxpNO4cw9ZhHxezdmzR07nns/fi6X1s6lItVMbPcRNdxy8scyYjRmoMilWuav\nwDxgBLAeuAoIAKjq9alSyGuB4/BKIS9S1W7LYKxaxmTjui5ODue8c52nqr26kSrb9r3dpzG9kbc6\nd1U9u5vnFfhcD2IzplO5JOyezOttEs62vSV2MxhYbxljjClCltyNMaYIWXI3xpgiZMndGGOKkCV3\nY4wpQpbcjTGmCFlyN8aYImTJ3RhjipAld2OMKUKW3I0xpghZcjfGmCJkyd0YY4qQJXdjjClCltyN\nMaYIWXI3xpgiZMndGGOKkCV3Y4wpQpbcjTGmCFlyN8aYImTJ3RhjipAld2OMKUKW3I0xpghZcjfG\nmCLkL3QAg8kHy1az5KllVNQMY/8T5xAMBQodkjHGZGXJPQeu6/Lz//kd/73jOQAcv49AyM/PHruK\nKbMmFTg6Y4zJZKdlcvDk357lqbuep6U5RktzjOaGZuo3NfDtU3+CqhY6PGOMyWDJPQcP3PAI0e0t\nGePbNtSx4vWVBYjIGGO6Zsk9B7GWeNZxcRxiLYl+jsYYY7pnyT0HR51zCKGSYMZ4IOhnl30m939A\nxhjTDUvuOTjx4mOYutdkwmVhAAIhP6GSEN+47Yv4/L4CR2eMMZmsWiYHwXCQXz71XV54YDGLH3mV\n4WOqOPbCI6gZP7zQoRljTFaW3HPk8/s4+NS5HHzq3LTxWDTGshfeIVQSZNfaaTiOfRkyxhReTsld\nRI4DrgF8wE2qenWH5+cB9wHvp4buUdXv5jHOAenJvz/LLz59PeII6ipllaX84F/fsNp3Y0zBdZvc\nRcQHXAccA6wGForI/ar6RoepT6vq/D6IcUBa+eYafvbJ39LSHGsda26M8tWjv8vfVt+AP2Bfiowx\nhZPLOYS5wHJVfU9VY8DfgFP6NqyB76GbHyMRT2aMx6NxFj/yWgEiMsaYNrkk93HAqnaPV6fGOjpI\nRF4TkYdEZI9sOxKRi0VkkYgs2rhx406EO3BsW19HMpGZ3FWVhi2NBYjIGGPa5Ovq30vARFXdC/gN\n8I9sk1T1RlWtVdXampqaPB26MPY/cQ7h0nDGeCKeZK/DZxYgImOMaZNLcl8DTGj3eHxqrJWq1qtq\nY+rnB4GAiIzIW5QD0CEf258psyYSKgm1joVLQ3z0f49n5ISifunGmEEgl6t+C4HpIjIFL6mfBZzT\nfoKIjAbWq6qKyFy8N43N+Q52IPEH/Pzsif/jP398kif//iyRsjDzLzmWucfPLnRoxhjTfXJX1YSI\nfB54GK8U8hZVXSoil6Sevx44HbhURBJAM3CWDtB2iU0NzTRubWT4uGp8vs7vLm2sa+TlR19n1/2m\nMWqidwrJdV02rd5MaUUJpRWlBEMBTrz4aA6YP4dgOMiw4eVdHrt+SwMtTTFGjKtGRPL6uowxpj0p\nVA6ura3VRYsW9dvxok0t/PLi63n67gU4PiEUCfHZay7iqHMOzZh78T5f5v3X2ro9lleX8blfX8T1\nl91Kc0MzrutywEm1nHjxMVxzyY1s/nALrqvsfsCuXHH7Fxkxtjptf9s21vGjc3/NkqfeQByhcmQF\nX/3D59jniD37/HUbY4qLiCxW1dpu5w2V5P7dM3/OggcWE4u2dXgMlQT5wQNXsPe8tuKeK074AQv/\n/Uq3+/MH/SQTSdRt+/05PofRU0byhzevab1TVVW5ZM5X+WDp6rTqmlBJiBte+SnjdhmTj5dnjBki\nck3uQ+Je+W0b63jhn+mJHaClKcZff3RP2tjCh7tP7ACJWCItsQO4SZet67ex5KllrWNvL36PD5ev\nyyibTMQT3P/bh3vyMowxJmdDIrlvWbuNQDD75YV1KzakD/T2i4zChlWbWh9uWLkJx5f5a07Gk6x5\nZ20vD2aMMdkNieQ+dpfRuK6bMe74HGYdsnvamM/fu1+Jm3TZtXZa6+Nd951KIpa5oEcoEkw7HWSM\nMfk0JJJ7uCTEed8+I60m3XGEcGmIc755Wtrc8686M+s+HJ/gOG0VLqGSIKUVJfjbfSMIlQSZe+Ic\nJu0+vnVs1KQajjj7kLRj+wI+yqpKOeFTR/X6tRljTDZDprvVx796CqMnj+RvV9/LlnXbmHXo7lz4\nvbMYM3VU2rxzv3kaPr/Dn666g0QsgTjC0ecfxgVXnckfv/13Xn5sCRU15Zz5lVPY77h9+Mv37uKZ\nexcQioQ46ZJjOPULJ2Qc+7LfX8L0fady37UP0dQQ5aCTazn/22dQWlHaXy/fGDPEDJlqGWOMKQa5\nVssMmU/uiUSCb57wI15+7DVUIRgJ8pmfXcDjtz/N0mffap130Km1nPjpY7n+y39i9dsfUjWqknOv\nPI2TLjk248ajD99dx7Wfv5mXHltCIOTn6PMO4+Kfnk+kLNLfL88YY9IMmU/un5n9Vd57dUVOcx3H\nSbsAGy4Jcf7/ncmZXzm5dax+SwMXzvhfGrduby2JDIQC7Fo7lV8+9T27A9UY0yeszr2dDSs35pzY\ngYzKmmhTC7f/4O60WvV/3/IEsaZYWq17vCXOu6+s4O1F7/Y6ZmOM6Y0hkdzffHF5r/cRi8Zp2NrW\np335y++nrcK0g4iwctmajHFjjOlPQyK57zZ3l17vIxDyU15V1vp4l30mE4oEM+apKhN2G9vr4xlj\nTG8MieQ+cmINk/ec0P3ElB19YXYIl4Y4+xsfxedv6yJ53CePJBgJpp1bD4T8TJk1kRn79f7NxBhj\nemNIJHeA6xb92LsjNJWLg+EAl/ziE+y2f3oi3n/+HL7zj8sZt+sYEKgaVcH//PAcPn75qWnzhg0v\n59fP/5DZR83C8TkEI0GOPv9wrn74W3Yx1RhTcEOmWqY913UzPp0nk8mM/u6qmlOiznWeMcb0VlHW\nuddvaeD2H9zDM/csIBQJMv/SYzn5sx/JSMqJRIKfXngdT9/9AsmEy6Q9JvDNv36Jn3zi2tZKFsfn\n8Kmrz+WuXzzAlrVbW7edNHM8Y3YZxQv3L24dG7frGD5y4RHc8s3bWxuL1UwYzm9e/BF3/vR+nrm7\n63hc1+XB3z/GP659kOaGKAedvB/nXHkaVSMr+ug3ZYwZ6gbNJ/doUwsX7/VlNq3eTDzViCtUEuKg\nk2u54vYvpc29aPcvsvqtD/Mab2cCQX9aPAeetC/f/Ov/S5vzy4uv5/HbnyHa1AKAP+CjclQFN73+\nS0qHlfRLnMaY4lB0de6P3/Y0W9dva02kAC1NLTx730JWvdVWevjG82/1W2IHMuJ57v5FrHyzLZ4N\nKzfyyF+eak3sAIl4koYtjTx002P9FqcxZmgZNMn9lSeXEt3ekjHu8zlpdezP3rewP8PK4PM5vLWw\nLZ63F7+XtZd8S1OMV554vT9DM8YMIYMmuY+dNir7ghsCIyeMaH04afdx/RhVFgI144e3PqwZPxzX\nzTz15fP7GDttdH9GZowZQgZNcj/h00fjC6RfqHR8DlUjK5h1WNuCG0effziBUKC/w0uLZ6/DZ7aO\n7Vo7jTFTRqbVyIN3rv7kzx3X3yEaY4aIQZPcR04YwQ8f/CajJtUQjAQJhPzsfsB0fvbEd9LKGh3H\n4bqFV1NWWdpuTPjYl05EfOnliiOn1mQ9Vsd5ANVjKzPGLvnVhd3GIyL8+D/fYtahuxMI+QlFgowY\nV813/nE546fb4tjGmL4xaKpldlBVNqzcRDAS7LaUcNVba9i2sZ49DprRmnBff3YZby18l49cOI+y\nSq+dwLP3LeChmx7n4187hVmHeJ+6P3hjJff/9j8c8rG5zD5yLwDq6rZzz8/uZ7cDpnPgibU9jqdu\nUz3NjVFGTaqxunhjzE7JtVpm0CX3jlSVZ+5ZwL//8ATquhxzwTwOO+OAjFrzznz47jru/tW/eH/J\nB+y+/6589H+P580Xl/OTT1xLc2MUf9DHOVeexvlXntHrWI0xpreGTHL/8Sd+wzP3LGitpAmXhqj9\nyD58+84vd/vpeNmCd7j86O8Sb4mTTCS9C7YC8ZbMBa0PPGU/vnvv5b2O1xhjeqPo6tyzeeel93j6\n7gVpJZLR7S0sevgVlj77Zrfb//IzNxDdHm3t0x6PJbImdoDn71tIIpH9OWOMGWgGdXJ/+bElJOOZ\nCTfa1MLiR1/rcttYS5wPXl/Zo+O9+sTSHs03xphCGdTJvayqDH+W2vdgKJDWez0bn9/BF+hZa53R\nU0b2aL4xxhTKoE7uh51+QNZxcRyOOOvgLrf1+Xwcfd5hBMO51cSLTxi3i5UuGmMGh0Gd3MsqS/n+\nA9+gvLqMkmERSoZFKK0o4aq7v0LVqMy69I4+e81F7D1vD0KRIKUVJQTCAQ4788CMm6UAfv3cD/vi\nJRhjTJ8Y9NUyAIl4gjeefxt1lZkH7Uog2LM7VNcsX8uHy9cxaY8Jra0MHr39af7zxyeYddjuVgZp\njBkw8loKKSLHAdcAPuAmVb26w/OSev4EoAm4UFVf6mqfhVyswxhjBqu8lUKKiA+4DjgemAmcLSIz\nO0w7Hpie+ncx8LseR2yMMSZvcjnnPhdYrqrvqWoM+BtwSoc5pwC3qucFoFJE7OqjMcYUSC7JfRyw\nqt3j1amxns4xxhjTT/q1WkZELhaRRSKyaOPGjf15aGOMGVJySe5rgAntHo9PjfV0Dqp6o6rWqmpt\nTU32drvGGGN6L5fkvhCYLiJTRCQInAXc32HO/cAF4jkAqFPVtXmO1RhjTI66vf9eVRMi8nngYbxS\nyFtUdamIXJJ6/nrgQbwyyOV4pZAXdbffxYsXbxKRD3oR+whgUy+2H0iK6bVAcb2eYnotUFyvZ6i+\nlkm5TCrYTUy9JSKLcqn1HAyK6bVAcb2eYnotUFyvx15L1wZ1+wFjjDHZWXI3xpgiNJiT+42FDiCP\nium1QHG9nmJ6LVBcr8deSxcG7Tl3Y4wxnRvMn9yNMcZ0YtAldxG5RUQ2iMjrhY6lt0Rkgog8ISJv\niMhSEflioWPaWSISFpEXReTV1Gv5TqFj6i0R8YnIyyLyQKFj6S0RWSEiS0TkFREZ9O1YRaRSRO4S\nkTdFZJmIHFjomHaGiMxI/T/Z8a9eRL6Ul30PttMyInIY0IjXqGzPQsfTG6nmamNU9SURKQcWA6eq\n6hsFDq3HUm2fS1W1UUQCwDPAF1ON5AYlEbkMqAWGqer8QsfTGyKyAqhV1aKoCxeRPwFPq+pNqZsr\nS1R1W6Hj6o1UB941wP6q2pt7gIBB+MldVZ8CthQ6jnxQ1bU7+t6ragOwjEHacC3VEbQx9TCQ+je4\nPjm0IyLjgROBmwodi0knIhXAYcDNAKoaG+yJPeUo4N18JHYYhMm9WInIZGA2sKCwkey81GmMV4AN\nwCOqOmhfC/Ar4HLALXQgeaLAoyKyWEQuLnQwvTQF2Aj8IXXa7CYRKS10UHlwFvDXfO3MkvsAICJl\nwN3Al1S1vtDx7CxVTarqPniN4+aKyKA8bSYi84ENqrq40LHk0SGp/zfHA59Lnd4crPzAHOB3qjob\n2A58vbAh9U7q1NLJwJ352qcl9wJLnZ++G7hNVe8pdDz5kPqK/ARwXKFj2UkHAyenzlP/DThSRP5S\n2JB6R1XXpP67AbgXbxGewWo1sLrdN8O78JL9YHY88JKqrs/XDi25F1DqIuTNwDJV/UWh4+kNEakR\nkcrUzxHgGODNwka1c1T1G6o6XlUn431VflxVzytwWDtNREpTF+xJnb44Fhi01Waqug5YJSIzUkNH\nAYOuCKGDs8njKRnIoSvkQCMifwXmASNEZDVwlareXNiodtrBwPnAktS5aoArVPXBAsa0s8YAf0pd\n8XeAO1R10JcQFolRwL3eZwn8wO2q+u/ChtRrXwBuS53OeI8cOtEOVKk33GOAz+R1v4OtFNIYY0z3\n7LSMMcYUIUvuxhhThCy5G2NMEbLkbowxRciSuzHGFCFL7sYYU4QsuRtjTBGy5G6MMUXo/wNFrui3\nT58rSwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d51b72b198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X[:,0], X[:,1], c = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1, 2)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:0][:, np.newaxis].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], shape=(0, 2), dtype=float64)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150,)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:,0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* What Linear Regression training algorithm can you use if you have a training set with millions of features?\n",
    "\n",
    "* Suppose the features in your training set have very different scales. What algorithms might suffer from this, and how? What can you do about it?\n",
    "\n",
    "* Can Gradient Descent get stuck in a local minimum when training a Logistic Regression model?\n",
    "\n",
    "* Do all Gradient Descent algorithms lead to the same model provided you let them run long enough?\n",
    "\n",
    "* Suppose you use Batch Gradient Descent and you plot the validation error at every epoch. If you notice that the validation error consistently goes up, what is likely going on? How can you fix this?\n",
    "\n",
    "* Suppose you are using Polynomial Regression. You plot the learning curves and you notice that there is a large gap between the training error and the validation error. What is happening? What are three ways to solve this?\n",
    "\n",
    "Suppose you are using Ridge Regression and you notice that the training error and the validation error are almost equal and fairly high. Would you say that the model suffers from high bias or high variance? Should you increase the regularization hyperparameter α or reduce it?\n",
    "\n",
    "Why would you want to use:\n",
    "\n",
    "Ridge Regression instead of Linear Regression?\n",
    "\n",
    "Lasso instead of Ridge Regression?\n",
    "\n",
    "Elastic Net instead of Lasso?\n",
    "\n",
    "Read about bias & variance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Regularization\n",
    "* Prevent overfitting\n",
    "* For linear model, regularization is acheived by constraining the weights of model\n",
    "* Ridge Regression, Lasso Regression & ElasticNet \n",
    "\n",
    "* Rdge Regression - Add regularization term to cost function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "idge_reg = Ridge(alpha=0.1, solver=\"cholesky\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lasso Regression\n",
    "Least Absolute Shrinkage and Selection Operator Regression (simply called Lasso Regression) is another regularized version of Linear Regression: just like Ridge Regression, it adds a regularization term to the cost function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "lasso_reg = Lasso(alpha=0.1)\n",
    "#lasso_reg.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ElasticNet\n",
    "* Combine them both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import ElasticNet\n",
    "elastic_net = ElasticNet(alpha=0.1, l1_ratio=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}