Pandas_and_regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pandas.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "FR_O1FkliAeB",
        "_7TCErRVgrRm",
        "zDVEvDLmt9cO",
        "l-XP7k1gGyVh",
        "6BVkbj2oq6o1"
      ],
      "authorship_tag": "ABX9TyOiZQU3qFrsrgb3pNaOyeve",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/olgasem10/Compling-tasks/blob/master/Pandas_and_regression.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJOOLQIp0qAC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import datetime\n",
        "from google.colab import files\n",
        "import numpy as np\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.model_selection import train_test_split\n",
        "from string import punctuation\n",
        "punctuation += '«»—–…“”'\n",
        "from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from scipy.sparse import hstack\n",
        "from sklearn.linear_model import SGDRegressor"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgo85ziA2TZC",
        "colab_type": "code",
        "outputId": "da677922-cbda-47f3-fdf3-1a0521345699",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-c273dedc-fe38-479c-b4ff-d979b61b090d\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-c273dedc-fe38-479c-b4ff-d979b61b090d\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving weatherHistory.csv to weatherHistory.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ulC5qf1u3HLb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weather_hist = pd.read_csv('weatherHistory.csv')\n",
        "weather_hist.rename({'Formatted Date':'date', 'Temperature (C)':'temp', 'Apparent Temperature (C)':'temp_app',\n",
        "                    'Precip Type':'weather_type', 'Wind Speed (km/h)':'wind_speed', 'Humidity':'humidity',\n",
        "                    'Daily Summary':'summary'}, \n",
        "                    axis=1, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVuiKs6QHCJ7",
        "colab_type": "code",
        "outputId": "735ea461-7a4a-44bf-a201-20f2b5e38cb5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "weather_hist.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Summary</th>\n",
              "      <th>weather_type</th>\n",
              "      <th>temp</th>\n",
              "      <th>temp_app</th>\n",
              "      <th>humidity</th>\n",
              "      <th>wind_speed</th>\n",
              "      <th>Wind Bearing (degrees)</th>\n",
              "      <th>Visibility (km)</th>\n",
              "      <th>Loud Cover</th>\n",
              "      <th>Pressure (millibars)</th>\n",
              "      <th>summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2006-04-01 00:00:00.000 +0200</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "      <td>rain</td>\n",
              "      <td>9.472222</td>\n",
              "      <td>7.388889</td>\n",
              "      <td>0.89</td>\n",
              "      <td>14.1197</td>\n",
              "      <td>251.0</td>\n",
              "      <td>15.8263</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1015.13</td>\n",
              "      <td>Partly cloudy throughout the day.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2006-04-01 01:00:00.000 +0200</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "      <td>rain</td>\n",
              "      <td>9.355556</td>\n",
              "      <td>7.227778</td>\n",
              "      <td>0.86</td>\n",
              "      <td>14.2646</td>\n",
              "      <td>259.0</td>\n",
              "      <td>15.8263</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1015.63</td>\n",
              "      <td>Partly cloudy throughout the day.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2006-04-01 02:00:00.000 +0200</td>\n",
              "      <td>Mostly Cloudy</td>\n",
              "      <td>rain</td>\n",
              "      <td>9.377778</td>\n",
              "      <td>9.377778</td>\n",
              "      <td>0.89</td>\n",
              "      <td>3.9284</td>\n",
              "      <td>204.0</td>\n",
              "      <td>14.9569</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1015.94</td>\n",
              "      <td>Partly cloudy throughout the day.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2006-04-01 03:00:00.000 +0200</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "      <td>rain</td>\n",
              "      <td>8.288889</td>\n",
              "      <td>5.944444</td>\n",
              "      <td>0.83</td>\n",
              "      <td>14.1036</td>\n",
              "      <td>269.0</td>\n",
              "      <td>15.8263</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1016.41</td>\n",
              "      <td>Partly cloudy throughout the day.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2006-04-01 04:00:00.000 +0200</td>\n",
              "      <td>Mostly Cloudy</td>\n",
              "      <td>rain</td>\n",
              "      <td>8.755556</td>\n",
              "      <td>6.977778</td>\n",
              "      <td>0.83</td>\n",
              "      <td>11.0446</td>\n",
              "      <td>259.0</td>\n",
              "      <td>15.8263</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1016.51</td>\n",
              "      <td>Partly cloudy throughout the day.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                            date  ...                            summary\n",
              "0  2006-04-01 00:00:00.000 +0200  ...  Partly cloudy throughout the day.\n",
              "1  2006-04-01 01:00:00.000 +0200  ...  Partly cloudy throughout the day.\n",
              "2  2006-04-01 02:00:00.000 +0200  ...  Partly cloudy throughout the day.\n",
              "3  2006-04-01 03:00:00.000 +0200  ...  Partly cloudy throughout the day.\n",
              "4  2006-04-01 04:00:00.000 +0200  ...  Partly cloudy throughout the day.\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FR_O1FkliAeB",
        "colab_type": "text"
      },
      "source": [
        "# Самый холодный и самый ветреный дни"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uF3d8XvaitXh",
        "colab_type": "code",
        "outputId": "ab3c0e18-f9ec-414f-922a-13047d51dd8b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "weather_hist.loc[weather_hist['temp'].idxmin()]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "date                                              2012-02-10 07:00:00+01:00\n",
              "Summary                                                               Foggy\n",
              "weather_type                                                           snow\n",
              "temp                                                               -21.8222\n",
              "temp_app                                                           -21.8222\n",
              "humidity                                                                0.8\n",
              "wind_speed                                                           3.0751\n",
              "Wind Bearing (degrees)                                                  323\n",
              "Visibility (km)                                                      1.3685\n",
              "Loud Cover                                                                0\n",
              "Pressure (millibars)                                                1033.66\n",
              "summary                   Foggy starting in the morning continuing until...\n",
              "Name: 54847, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JUByEeWmiCj6",
        "colab_type": "code",
        "outputId": "ec285fa7-e70d-4e70-eb61-121d884c2f09",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "weather_hist.loc[weather_hist['wind_speed'].idxmax()]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "date                                              2007-01-29 12:00:00+00:00\n",
              "Summary                                 Dangerously Windy and Partly Cloudy\n",
              "weather_type                                                           rain\n",
              "temp                                                                8.94444\n",
              "temp_app                                                            3.48333\n",
              "humidity                                                               0.49\n",
              "wind_speed                                                          63.8526\n",
              "Wind Bearing (degrees)                                                  307\n",
              "Visibility (km)                                                     11.4471\n",
              "Loud Cover                                                                0\n",
              "Pressure (millibars)                                                1009.05\n",
              "summary                   Mostly cloudy throughout the day and windy sta...\n",
              "Name: 12181, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 166
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_7TCErRVgrRm",
        "colab_type": "text"
      },
      "source": [
        "# Дождь летом 2014"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fKD6BSEtdMVL",
        "colab_type": "text"
      },
      "source": [
        "Посмотрим на то, какие существуют типы погоды"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jZ_gKkWLdWY7",
        "colab_type": "code",
        "outputId": "4e103a13-6d49-4c2e-9e3e-21256107f4d2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "weather_hist.Summary.unique()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Partly Cloudy', 'Mostly Cloudy', 'Overcast', 'Foggy',\n",
              "       'Breezy and Mostly Cloudy', 'Clear', 'Breezy and Partly Cloudy',\n",
              "       'Breezy and Overcast', 'Humid and Mostly Cloudy',\n",
              "       'Humid and Partly Cloudy', 'Windy and Foggy', 'Windy and Overcast',\n",
              "       'Breezy and Foggy', 'Windy and Partly Cloudy', 'Breezy',\n",
              "       'Dry and Partly Cloudy', 'Windy and Mostly Cloudy',\n",
              "       'Dangerously Windy and Partly Cloudy', 'Dry', 'Windy',\n",
              "       'Humid and Overcast', 'Light Rain', 'Drizzle', 'Windy and Dry',\n",
              "       'Dry and Mostly Cloudy', 'Breezy and Dry', 'Rain'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QU4V-fGCdZzl",
        "colab_type": "text"
      },
      "source": [
        "Будем считать, что дождь соответствует погоде 'Rain', 'Light Rain', 'Drizzle'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YgVTwovb4a5j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weather_hist['date'] =  pd.to_datetime(weather_hist['date'])\n",
        "weather = weather_hist[['date', 'Summary']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DQgp99uFOVow",
        "colab_type": "code",
        "outputId": "a64b64fd-53a5-4eb4-96c7-1c4266c91bdf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        }
      },
      "source": [
        "d1 = datetime.datetime(2014, 6, 1).replace(tzinfo=datetime.timezone.utc)\n",
        "d2 = datetime.datetime(2014, 8, 30).replace(tzinfo=datetime.timezone.utc)\n",
        "mask = (weather['date'] >= d1) & (weather['date'] <= d2)\n",
        "weather = weather.loc[mask]\n",
        "weather"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>70872</th>\n",
              "      <td>2014-08-01 00:00:00+02:00</td>\n",
              "      <td>Overcast</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70873</th>\n",
              "      <td>2014-08-01 01:00:00+02:00</td>\n",
              "      <td>Overcast</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70874</th>\n",
              "      <td>2014-08-01 02:00:00+02:00</td>\n",
              "      <td>Overcast</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70875</th>\n",
              "      <td>2014-08-01 03:00:00+02:00</td>\n",
              "      <td>Mostly Cloudy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70876</th>\n",
              "      <td>2014-08-01 04:00:00+02:00</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75232</th>\n",
              "      <td>2014-06-09 19:00:00+02:00</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75233</th>\n",
              "      <td>2014-06-09 20:00:00+02:00</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75234</th>\n",
              "      <td>2014-06-09 21:00:00+02:00</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75235</th>\n",
              "      <td>2014-06-09 22:00:00+02:00</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75236</th>\n",
              "      <td>2014-06-09 23:00:00+02:00</td>\n",
              "      <td>Partly Cloudy</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2158 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                            date        Summary\n",
              "70872  2014-08-01 00:00:00+02:00       Overcast\n",
              "70873  2014-08-01 01:00:00+02:00       Overcast\n",
              "70874  2014-08-01 02:00:00+02:00       Overcast\n",
              "70875  2014-08-01 03:00:00+02:00  Mostly Cloudy\n",
              "70876  2014-08-01 04:00:00+02:00  Partly Cloudy\n",
              "...                          ...            ...\n",
              "75232  2014-06-09 19:00:00+02:00  Partly Cloudy\n",
              "75233  2014-06-09 20:00:00+02:00  Partly Cloudy\n",
              "75234  2014-06-09 21:00:00+02:00  Partly Cloudy\n",
              "75235  2014-06-09 22:00:00+02:00  Partly Cloudy\n",
              "75236  2014-06-09 23:00:00+02:00  Partly Cloudy\n",
              "\n",
              "[2158 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nG6V374qKuJ8",
        "colab_type": "code",
        "outputId": "66a033bd-992e-4141-d1c5-e6840635611b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        }
      },
      "source": [
        "weather = weather.loc[weather['Summary'].isin(['Rain', 'Light Rain', 'Drizzle'])]\n",
        "weather"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>Summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [date, Summary]\n",
              "Index: []"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RZyzWdAffWlK",
        "colab_type": "text"
      },
      "source": [
        "К сожалению, согласно данным, дождь летом 2014 не шел ни разу"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zDVEvDLmt9cO",
        "colab_type": "text"
      },
      "source": [
        "# Год, месяц, неделя"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rLGOBeTOuLwH",
        "colab_type": "text"
      },
      "source": [
        "Самый ветренный месяц (месяц с наибольшей средней скоростью ветра) - март"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XjtGoXYurHZL",
        "colab_type": "code",
        "outputId": "e1430be3-ea6c-4dd1-f9f5-47fa5d9acb06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        }
      },
      "source": [
        "weather = weather_hist[['date', 'wind_speed']]\n",
        "months = weather.groupby(weather.date.dt.month).agg(['mean'])\n",
        "months.loc[months['wind_speed'].idxmax()]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>wind_speed</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>date</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>13.415825</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     wind_speed\n",
              "           mean\n",
              "date           \n",
              "3     13.415825"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 167
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YpUrCsnAjw3R",
        "colab_type": "text"
      },
      "source": [
        "Самый влажный год (год с наибольшей средней влажностью) - 2005"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wz9q83a4loQn",
        "colab_type": "code",
        "outputId": "12e9fc1c-abcc-4cf9-d403-04c5e0950a87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        }
      },
      "source": [
        "weather = weather_hist[['date', 'humidity']]\n",
        "years = weather.groupby(weather.date.dt.year).agg(['mean'])\n",
        "years.loc[years['humidity'].idxmax()]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>humidity</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>date</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2005</th>\n",
              "      <td>0.89</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     humidity\n",
              "         mean\n",
              "date         \n",
              "2005     0.89"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 155
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UZa1yydcuYL-",
        "colab_type": "text"
      },
      "source": [
        "Неделя с самой переменчивой погодой (неделя с наибольшим отклонением температуры) - шестая"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cG0U53pAjyGW",
        "colab_type": "code",
        "outputId": "816c6380-1b02-4e00-d835-882966163975",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        }
      },
      "source": [
        "weather = weather_hist[['date', 'temp']]\n",
        "weeks = weather.groupby(weather.date.dt.weekofyear).agg(['std'])\n",
        "weeks.loc[weeks['temp'].idxmax()]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr th {\n",
              "        text-align: left;\n",
              "    }\n",
              "\n",
              "    .dataframe thead tr:last-of-type th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>temp</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th></th>\n",
              "      <th>std</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>date</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>6.456311</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          temp\n",
              "           std\n",
              "date          \n",
              "6     6.456311"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 160
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l-XP7k1gGyVh",
        "colab_type": "text"
      },
      "source": [
        "# Признаки текста"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DlUJU1sRGxzh",
        "colab_type": "code",
        "outputId": "8a5d408b-08bb-4fab-de85-d5245870a745",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8723094a-5941-4563-9b0c-456633864da3\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-8723094a-5941-4563-9b0c-456633864da3\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving data.csv to data.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HzsVixI6RLW0",
        "colab_type": "code",
        "outputId": "19f45f13-8fe9-460c-afd9-852690df39f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y_train, y_test = train_test_split(data.deal_probability, random_state=42)\n",
        "np.random.rand(len(y_test))\n",
        "np.sqrt(mean_squared_error(y_test, np.random.rand(len(y_test))))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5278561191236458"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 387
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orf4zvtzRptb",
        "colab_type": "code",
        "outputId": "c7d0a955-6de7-4009-ad6d-00f4a5c38286",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "np.full(len(y_test), np.mean(y_train))\n",
        "np.sqrt(mean_squared_error(y_test, np.full(len(y_test), np.mean(y_train))))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.26367475849307315"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 388
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zNyIR0y0Opzc",
        "colab_type": "text"
      },
      "source": [
        "Добавим следующие признаки:\n",
        "\n",
        "1) Соотношение количества уникальных токенов к общему количеству (процент уникальных) - если в тексте объявления часто повторяются одни и те же слова, то это ухудшает его восприятие\n",
        "\n",
        "2) Наличие в тексте цифр может указывать на более точное описание параметров и характеристик товара\n",
        "\n",
        "3) Количество восклицательных знаков - чрезмерное их употребление может отталкивать\n",
        "\n",
        "4) Количество предложений - текст, разделенный на предложения воспринимается лучше\n",
        "\n",
        "5) Наличие в тексте английских символов - может указывать на названия фирменных товаров\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2psKsfLsR27o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data['text_len'] = data['description'].apply(len)\n",
        "data['text_len_tokens'] = data['description'].apply(lambda x: len(x.split()))\n",
        "data['number_of_caps'] = data['description'].apply(lambda x: len([ch for ch in x if ch.isupper()]))\n",
        "data['number_of_nonalphanum'] = data['description'].apply(lambda x: len([ch for ch in x if not ch.isalnum()]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VOF2JXYwXIEr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tokenizer(text):\n",
        "  text = text.split()\n",
        "  text = [txt.strip(punctuation) for txt in text]\n",
        "  text = [txt.lower() for txt in text if text]\n",
        "  return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KBRyFEngS3aL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eng = 'abcdefghijklmnopqrstuvwxyz'\n",
        "data['number_of_exclam_mark'] = data['description'].apply(lambda x: len([ch for ch in x if ch == '!']))\n",
        "data['unique_tokens'] = data['description'].apply(lambda x: len(set(tokenizer(x)))/len(tokenizer(x)))\n",
        "data['number_of_num'] = data['description'].apply(lambda x: len([ch for ch in x if ch.isnumeric()]))\n",
        "data['number_of_sent'] = data['description'].apply(lambda x: len(x.split('.')))\n",
        "data['number_of_eng_ch'] = data['description'].apply(lambda x: len([ch for ch in x if ch in eng]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d6ldN4nDnUZJ",
        "colab_type": "code",
        "outputId": "53a25f9e-0ae7-446c-a928-1b36fde9991c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "X = data[['text_len', 'text_len_tokens', 'number_of_caps', 'number_of_nonalphanum', \n",
        "           'number_of_exclam_mark', 'unique_tokens', 'number_of_eng_ch', 'number_of_num', 'number_of_sent']].values\n",
        "y = data.deal_probability.values\n",
        "X.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(138715, 9)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 403
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LQ5A4W1anfYx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fjEqa22rLzFi",
        "colab_type": "text"
      },
      "source": [
        "Качество улучшилось незначительно - средняя ошибка уменьшилась всего лишь на 0.0001"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BAsT4qQTnib8",
        "colab_type": "code",
        "outputId": "57f6cc11-70ad-48a6-9a60-4141cb1951e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "regressor = Ridge(alpha=1)\n",
        "regressor.fit(X_train, y_train)\n",
        "preds = regressor.predict(X_test)\n",
        "np.sqrt(mean_squared_error(y_test, preds))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.2632458527862207"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 405
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6BVkbj2oq6o1",
        "colab_type": "text"
      },
      "source": [
        "# Векторизация"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-3kAMeY4M1qa",
        "colab_type": "text"
      },
      "source": [
        "CountVectorizer:\n",
        "\n",
        "удалось уменьшить значение ошибки на 0,0005 за счет подбора уровня max_features, min_df и ngram_range\n",
        "изменение праметра analyzer ухудшало результат"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "03rke-epq88w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cv = CountVectorizer(max_features=2400, min_df=10, max_df=0.4, ngram_range=(1,1))\n",
        "X = cv.fit_transform(data.description)\n",
        "y = data.deal_probability.values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Olqtw9d9r0Eq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "igSs42AGr2xR",
        "colab_type": "code",
        "outputId": "046ab910-9065-443e-cb9b-425073f5c116",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "regressor = Ridge(alpha=10)\n",
        "regressor.fit(X_train, y_train)\n",
        "preds = regressor.predict(X_test)\n",
        "np.sqrt(mean_squared_error(y_test, preds))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.25069997449245585"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 245
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0vHW1pm0sVul",
        "colab_type": "text"
      },
      "source": [
        "TfidfVectorizer:\n",
        "\n",
        "удалось уменьшить значение ошибки на 0,0018 за счет подбора уровня max_features\n",
        "\n",
        "изменение праметров analyzer и ngram_range ухудшало результат"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "efJg0T-QsXqx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cv = TfidfVectorizer(max_features=2700, min_df=5, max_df=0.4)\n",
        "X = cv.fit_transform(data.description)\n",
        "y = data.deal_probability.values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YEVQ7J10sfHu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lvO92uNcsi60",
        "colab_type": "code",
        "outputId": "935869af-c7dd-4974-f536-91dca7ed291d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "regressor = Ridge(alpha=1)\n",
        "regressor.fit(X_train, y_train)\n",
        "preds = regressor.predict(X_test)\n",
        "np.sqrt(mean_squared_error(y_test, preds))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.246553621292103"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 339
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ai4s0mFptTfu",
        "colab_type": "text"
      },
      "source": [
        "SGDRegressor:\n",
        "\n",
        "удалось улучшить результат с 0.248 до 0.246 за счет увеличения параметров max_iter и n_iter_no_change (но это привело к увеличению времени работы)\n",
        "\n",
        "изменение параметра alpha влияло незначительно, изменение параметра loss ухудшало результат"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oBvHOM0QtW99",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cv = CountVectorizer(max_features=2500, min_df=10, max_df=0.4, ngram_range=(1,2))\n",
        "tfidf = TfidfVectorizer(max_features=2500, min_df=5,analyzer='char', ngram_range=(3,3), max_df=0.4)\n",
        "\n",
        "X_cv = cv.fit_transform(data.description)\n",
        "X_tfidf = tfidf.fit_transform(data.description)\n",
        "\n",
        "y = data.deal_probability.values\n",
        "\n",
        "X = hstack([X_cv, X_tfidf])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VkJeB-dTtmtN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vP6bxRXrtoPy",
        "colab_type": "code",
        "outputId": "8b0ff082-e3bf-4158-86d5-aa3cd09517c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "regressor = SGDRegressor(max_iter=200, alpha=0.0001, n_iter_no_change = 100)\n",
        "regressor.fit(X_train, y_train)\n",
        "preds = regressor.predict(X_test)\n",
        "np.sqrt(mean_squared_error(y_test, preds))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.24679546244592068"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 384
        }
      ]
    }
  ]
}