toxic_model.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "FNLCxH8dM_aI"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ['KAGGLE_CONFIG_DIR']='.'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BIlgHWoKNWmT",
        "outputId": "01f7ad1f-77ee-4827-c370-a3ac555be43a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 ./kaggle.json'\n",
            "Downloading jigsaw-toxic-comment-classification-challenge.zip to /content\n",
            " 99% 52.0M/52.6M [00:03<00:00, 24.3MB/s]\n",
            "100% 52.6M/52.6M [00:03<00:00, 16.4MB/s]\n"
          ]
        }
      ],
      "source": [
        "!kaggle competitions download -c jigsaw-toxic-comment-classification-challenge"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3_v3xqw6NaJ9",
        "outputId": "b33dcadd-ae0b-4b7f-dee3-8d3e7ea73a77"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['.config',\n",
              " 'kaggle.json',\n",
              " 'jigsaw-toxic-comment-classification-challenge.zip',\n",
              " 'sample_data']"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "os.listdir()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IM6kBkcANbfT",
        "outputId": "18923942-45e7-4ece-ece7-0e7d1ae775d4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  jigsaw-toxic-comment-classification-challenge.zip\n",
            "  inflating: data/sample_submission.csv.zip  \n",
            "  inflating: data/test.csv.zip       \n",
            "  inflating: data/test_labels.csv.zip  \n",
            "  inflating: data/train.csv.zip      \n"
          ]
        }
      ],
      "source": [
        "!unzip jigsaw-toxic-comment-classification-challenge.zip -d data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "gIZxjQzHNfzR"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "train_df=pd.read_csv('data/train.csv.zip')\n",
        "test_df=pd.read_csv('data/test.csv.zip')\n",
        "sub_df=pd.read_csv('data/sample_submission.csv.zip')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "CQbN6q0qyGDc",
        "outputId": "70de87eb-6cb7-43e5-a98e-5cb382d6454f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-78d377cf-7ed5-4ab5-b963-67142b45421a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0001b41b1c6bb37e</td>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0001d958c54c6e35</td>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159566</th>\n",
              "      <td>ffe987279560d7ff</td>\n",
              "      <td>\":::::And for the second time of asking, when ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159567</th>\n",
              "      <td>ffea4adeee384e90</td>\n",
              "      <td>You should be ashamed of yourself \\n\\nThat is ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159568</th>\n",
              "      <td>ffee36eab5c267c9</td>\n",
              "      <td>Spitzer \\n\\nUmm, theres no actual article for ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159569</th>\n",
              "      <td>fff125370e4aaaf3</td>\n",
              "      <td>And it looks like it was actually you who put ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159570</th>\n",
              "      <td>fff46fc426af1f9a</td>\n",
              "      <td>\"\\nAnd ... I really don't think you understand...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>159571 rows × 8 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-78d377cf-7ed5-4ab5-b963-67142b45421a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-78d377cf-7ed5-4ab5-b963-67142b45421a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-78d377cf-7ed5-4ab5-b963-67142b45421a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                      id                                       comment_text  \\\n",
              "0       0000997932d777bf  Explanation\\nWhy the edits made under my usern...   \n",
              "1       000103f0d9cfb60f  D'aww! He matches this background colour I'm s...   \n",
              "2       000113f07ec002fd  Hey man, I'm really not trying to edit war. It...   \n",
              "3       0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...   \n",
              "4       0001d958c54c6e35  You, sir, are my hero. Any chance you remember...   \n",
              "...                  ...                                                ...   \n",
              "159566  ffe987279560d7ff  \":::::And for the second time of asking, when ...   \n",
              "159567  ffea4adeee384e90  You should be ashamed of yourself \\n\\nThat is ...   \n",
              "159568  ffee36eab5c267c9  Spitzer \\n\\nUmm, theres no actual article for ...   \n",
              "159569  fff125370e4aaaf3  And it looks like it was actually you who put ...   \n",
              "159570  fff46fc426af1f9a  \"\\nAnd ... I really don't think you understand...   \n",
              "\n",
              "        toxic  severe_toxic  obscene  threat  insult  identity_hate  \n",
              "0           0             0        0       0       0              0  \n",
              "1           0             0        0       0       0              0  \n",
              "2           0             0        0       0       0              0  \n",
              "3           0             0        0       0       0              0  \n",
              "4           0             0        0       0       0              0  \n",
              "...       ...           ...      ...     ...     ...            ...  \n",
              "159566      0             0        0       0       0              0  \n",
              "159567      0             0        0       0       0              0  \n",
              "159568      0             0        0       0       0              0  \n",
              "159569      0             0        0       0       0              0  \n",
              "159570      0             0        0       0       0              0  \n",
              "\n",
              "[159571 rows x 8 columns]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "UffMzU-OyHZ4",
        "outputId": "7a23c1f3-6638-4c14-a29a-75fb8447addc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-40904c2c-2273-4796-9873-42e562d04e12\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00001cee341fdb12</td>\n",
              "      <td>Yo bitch Ja Rule is more succesful then you'll...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0000247867823ef7</td>\n",
              "      <td>== From RfC == \\n\\n The title is fine as it is...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>00013b17ad220c46</td>\n",
              "      <td>\" \\n\\n == Sources == \\n\\n * Zawe Ashton on Lap...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>00017563c3f7919a</td>\n",
              "      <td>:If you have a look back at the source, the in...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>00017695ad8997eb</td>\n",
              "      <td>I don't anonymously edit articles at all.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153159</th>\n",
              "      <td>fffcd0960ee309b5</td>\n",
              "      <td>. \\n i totally agree, this stuff is nothing bu...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153160</th>\n",
              "      <td>fffd7a9a6eb32c16</td>\n",
              "      <td>== Throw from out field to home plate. == \\n\\n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153161</th>\n",
              "      <td>fffda9e8d6fafa9e</td>\n",
              "      <td>\" \\n\\n == Okinotorishima categories == \\n\\n I ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153162</th>\n",
              "      <td>fffe8f1340a79fc2</td>\n",
              "      <td>\" \\n\\n == \"\"One of the founding nations of the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153163</th>\n",
              "      <td>ffffce3fb183ee80</td>\n",
              "      <td>\" \\n :::Stop already. Your bullshit is not wel...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>153164 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-40904c2c-2273-4796-9873-42e562d04e12')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-40904c2c-2273-4796-9873-42e562d04e12 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-40904c2c-2273-4796-9873-42e562d04e12');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                      id                                       comment_text\n",
              "0       00001cee341fdb12  Yo bitch Ja Rule is more succesful then you'll...\n",
              "1       0000247867823ef7  == From RfC == \\n\\n The title is fine as it is...\n",
              "2       00013b17ad220c46  \" \\n\\n == Sources == \\n\\n * Zawe Ashton on Lap...\n",
              "3       00017563c3f7919a  :If you have a look back at the source, the in...\n",
              "4       00017695ad8997eb          I don't anonymously edit articles at all.\n",
              "...                  ...                                                ...\n",
              "153159  fffcd0960ee309b5  . \\n i totally agree, this stuff is nothing bu...\n",
              "153160  fffd7a9a6eb32c16  == Throw from out field to home plate. == \\n\\n...\n",
              "153161  fffda9e8d6fafa9e  \" \\n\\n == Okinotorishima categories == \\n\\n I ...\n",
              "153162  fffe8f1340a79fc2  \" \\n\\n == \"\"One of the founding nations of the...\n",
              "153163  ffffce3fb183ee80  \" \\n :::Stop already. Your bullshit is not wel...\n",
              "\n",
              "[153164 rows x 2 columns]"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "Ce5k9m4SyQV-",
        "outputId": "3464db7d-75e4-431b-dacc-1921171c575c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-6573d66b-1de1-4b8c-8666-5088241d3cda\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00001cee341fdb12</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0000247867823ef7</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>00013b17ad220c46</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>00017563c3f7919a</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>00017695ad8997eb</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153159</th>\n",
              "      <td>fffcd0960ee309b5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153160</th>\n",
              "      <td>fffd7a9a6eb32c16</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153161</th>\n",
              "      <td>fffda9e8d6fafa9e</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153162</th>\n",
              "      <td>fffe8f1340a79fc2</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153163</th>\n",
              "      <td>ffffce3fb183ee80</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>153164 rows × 7 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6573d66b-1de1-4b8c-8666-5088241d3cda')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6573d66b-1de1-4b8c-8666-5088241d3cda button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6573d66b-1de1-4b8c-8666-5088241d3cda');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                      id  toxic  severe_toxic  obscene  threat  insult  \\\n",
              "0       00001cee341fdb12    0.5           0.5      0.5     0.5     0.5   \n",
              "1       0000247867823ef7    0.5           0.5      0.5     0.5     0.5   \n",
              "2       00013b17ad220c46    0.5           0.5      0.5     0.5     0.5   \n",
              "3       00017563c3f7919a    0.5           0.5      0.5     0.5     0.5   \n",
              "4       00017695ad8997eb    0.5           0.5      0.5     0.5     0.5   \n",
              "...                  ...    ...           ...      ...     ...     ...   \n",
              "153159  fffcd0960ee309b5    0.5           0.5      0.5     0.5     0.5   \n",
              "153160  fffd7a9a6eb32c16    0.5           0.5      0.5     0.5     0.5   \n",
              "153161  fffda9e8d6fafa9e    0.5           0.5      0.5     0.5     0.5   \n",
              "153162  fffe8f1340a79fc2    0.5           0.5      0.5     0.5     0.5   \n",
              "153163  ffffce3fb183ee80    0.5           0.5      0.5     0.5     0.5   \n",
              "\n",
              "        identity_hate  \n",
              "0                 0.5  \n",
              "1                 0.5  \n",
              "2                 0.5  \n",
              "3                 0.5  \n",
              "4                 0.5  \n",
              "...               ...  \n",
              "153159            0.5  \n",
              "153160            0.5  \n",
              "153161            0.5  \n",
              "153162            0.5  \n",
              "153163            0.5  \n",
              "\n",
              "[153164 rows x 7 columns]"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sub_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "iSgebKVEQ_Tk"
      },
      "outputs": [],
      "source": [
        "train_df=train_df.drop(['id'],axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "N97vK5OfNhQR",
        "outputId": "ddef00f1-a40e-4eab-e959-09b661cb2024"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-7a5f7145-3afd-4534-97df-366cf409ef2f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>11042</th>\n",
              "      <td>Donald Goerke was the inventor. He recently pa...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8306</th>\n",
              "      <td>I sincerely wish i could meet you chumps   \\n ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49977</th>\n",
              "      <td>I don't now why you said that I am a pro-frenc...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29250</th>\n",
              "      <td>Epic Fail ==\\nFAIL FAIL FAIL\\n\\n==</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40840</th>\n",
              "      <td>(Added new section showing verifiable links th...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>114467</th>\n",
              "      <td>\"\\n\\n Commons \\n\\nYou've got a message there, ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>104841</th>\n",
              "      <td>Stop pretending to be an admin \\n\\nGo cry to o...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74627</th>\n",
              "      <td>They are not random complaints. And I am not f...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>129369</th>\n",
              "      <td>\"\\nMore ridiculous wikilawyering (as in \"\"misi...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40471</th>\n",
              "      <td>(UTC)\\nThe World is The Pictorial World (see t...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7a5f7145-3afd-4534-97df-366cf409ef2f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7a5f7145-3afd-4534-97df-366cf409ef2f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7a5f7145-3afd-4534-97df-366cf409ef2f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                             comment_text  toxic  \\\n",
              "11042   Donald Goerke was the inventor. He recently pa...      0   \n",
              "8306    I sincerely wish i could meet you chumps   \\n ...      0   \n",
              "49977   I don't now why you said that I am a pro-frenc...      0   \n",
              "29250                  Epic Fail ==\\nFAIL FAIL FAIL\\n\\n==      0   \n",
              "40840   (Added new section showing verifiable links th...      0   \n",
              "114467  \"\\n\\n Commons \\n\\nYou've got a message there, ...      0   \n",
              "104841  Stop pretending to be an admin \\n\\nGo cry to o...      1   \n",
              "74627   They are not random complaints. And I am not f...      0   \n",
              "129369  \"\\nMore ridiculous wikilawyering (as in \"\"misi...      0   \n",
              "40471   (UTC)\\nThe World is The Pictorial World (see t...      0   \n",
              "\n",
              "        severe_toxic  obscene  threat  insult  identity_hate  \n",
              "11042              0        0       0       0              0  \n",
              "8306               0        0       0       0              0  \n",
              "49977              0        0       0       0              0  \n",
              "29250              0        0       0       0              0  \n",
              "40840              0        0       0       0              0  \n",
              "114467             0        0       0       0              0  \n",
              "104841             0        0       0       0              0  \n",
              "74627              0        0       0       0              0  \n",
              "129369             0        0       0       0              0  \n",
              "40471              0        0       0       0              0  "
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_df.sample(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xZ1VeLmQOgyV",
        "outputId": "3c057aef-2915-43ee-e58f-4aef0aaed913"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m92.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m80.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install transformers --quiet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "lpnQsvctNwFR"
      },
      "outputs": [],
      "source": [
        "import transformers\n",
        "import torch\n",
        "from transformers import DistilBertModel,DistilBertTokenizer,DistilBertConfig"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "91c3e20384384fc1aa57b21c64a1909a",
            "a7ccc548d0444837a21cfcd9b9474a42",
            "68f7235619834d9d9f1d164a63319824",
            "07efa77917f44b9a8f7416ee4bc1ca0e",
            "58f1754a3f954dc9a582bec5d63baae5",
            "21478fea93704608b18523074e9c81bf",
            "e07506c478a846f1ab24591953b5a312",
            "2262ba8c53be48aebed1f162791ef9f6",
            "dabb0d270e5a4e088a977f3a144c3a18",
            "10a4b3ac62a94f01942fd3ce8af68cfc",
            "645c94627ff542119525b77c9ad172fb",
            "3b4f9a31690f4513bc557c31113cb3fb",
            "5a965e00ff5146539b930ee7af62f6b3",
            "4f3b42b77ebd4cf4bc11f88653801e82",
            "0b6ad200b4924ebda9509fa3be12caa0",
            "c056b40910a14abab93beac0d8f5fef8",
            "f8f4c51eb03647eabc125ad6f19bdc8f",
            "be25c816264147f3a774d1cd977a6ddb",
            "59cbef76baf64a5183377873d0dd4d05",
            "11a9fb67f1e64443a0b679440483c947",
            "6ea453dc7eec409e86d119fb5d40ac2e",
            "3a71d2dc01a4473fbc71a7fdf1cb2afe",
            "dcfd001961204b1685f4657205cb50e3",
            "fee9288216d1422e96b54b140578ec14",
            "c55e15328e3444a88b44fb00cf19cd60",
            "b7c7dad6dcb04d6fba4b22f646f6ed78",
            "f743507a96194537aec2a3fcda764c85",
            "8ba1b98098ee49abb81c64ba863f8805",
            "a520f5c368884575a7b9be38f0b0b3e3",
            "1328d06b169c43c7a5d0a9ea5a1bcf8c",
            "273ad16d3ff34175abe8b249951ddfe6",
            "de30acd76ffe406b8f63d2f26a86bd6f",
            "c15beb3e41d54972a30f8cfd2d21ad72"
          ]
        },
        "id": "O0wxou1_OeB6",
        "outputId": "627bbe10-2701-4d2b-fbc8-d1e80a335956"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "91c3e20384384fc1aa57b21c64a1909a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3b4f9a31690f4513bc557c31113cb3fb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dcfd001961204b1685f4657205cb50e3",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/483 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tokenizer=DistilBertTokenizer.from_pretrained('distilbert-base-uncased',truncation=True,do_lower_Case=True) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zCupOP3XPip_",
        "outputId": "b2e07ecf-dfcb-436c-b395-e6e93b3c2a58"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "{'input_ids': [101, 12139, 2006, 2023, 3395, 1998, 2029, 2024, 10043, 2000, 2216, 1997, 4241, 15909, 25619, 5004, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tokenizer.encode_plus(train_df.comment_text[9],None,max_length=100,add_special_tokens=True,pad_to_max_length=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v0iGKmMj2jBC",
        "outputId": "c5da3971-dd0c-4547-c1a4-722ee4bb251a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "comment_text     Explanation\\nWhy the edits made under my usern...\n",
              "toxic                                                            0\n",
              "severe_toxic                                                     0\n",
              "obscene                                                          0\n",
              "threat                                                           0\n",
              "insult                                                           0\n",
              "identity_hate                                                    0\n",
              "Name: 0, dtype: object"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_df.loc[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "vUkhZV7R3TV5"
      },
      "outputs": [],
      "source": [
        "cols=['toxic',\t'severe_toxic','obscene','threat',\t'insult',\t'identity_hate']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vhGSYA2O1pme",
        "outputId": "fe39bcc2-4750-4176-8369-be15457a2bcc"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([0, 0, 0, 0, 0, 0])"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_df[cols].values[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "E2tYH8NkP63F"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "QBuJBXPaSrlp"
      },
      "outputs": [],
      "source": [
        "class JigsawDataset(Dataset):\n",
        "  def __init__(self,df,tokenizer,max_length,test=False):\n",
        "    self.df=df\n",
        "    self.tokenizer=tokenizer\n",
        "    self.max_length=max_length\n",
        "    self.test=test\n",
        "    # self.targets=df[cols].values\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.df)\n",
        "\n",
        "  def __getitem__(self,index):\n",
        "    text=self.df.comment_text.values[index]\n",
        "    inputs=self.tokenizer(text,\n",
        "                          max_length=self.max_length,\n",
        "                          add_special_tokens=True,\n",
        "                          pad_to_max_length=True,\n",
        "                          return_tensors='pt'\n",
        "                          )\n",
        "    # outputs={\n",
        "    #     'ids':torch.tensor(inputs['input_ids'],dtype=torch.long),\n",
        "    #     'masks':torch.tensor(inputs['attention_mask'],dtype=torch.long),\n",
        "    # }\n",
        "    if self.test is True:\n",
        "      outputs=torch.tensor([0,0,0,0,0,0],dtype=torch.float)\n",
        "    else:\n",
        "      outputs=torch.tensor(self.df[cols].values[index].tolist(),dtype=torch.float)\n",
        "    return inputs , outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "yKEd1BmA76wU"
      },
      "outputs": [],
      "source": [
        "def get_default_device():\n",
        "    \"\"\"Pick GPU if available, else CPU\"\"\"\n",
        "    if torch.cuda.is_available():\n",
        "        return torch.device('cuda')\n",
        "    else:\n",
        "        return torch.device('cpu')\n",
        "device=get_default_device()\n",
        "    \n",
        "def to_device(data, device):\n",
        "    \"\"\"Move tensor(s) to chosen device\"\"\"\n",
        "    if isinstance(data, (list,tuple)):\n",
        "        return [to_device(x, device) for x in data]\n",
        "    # return [key:value.to(device) for key,value in data.items()]\n",
        "\n",
        "class DeviceDataLoader():\n",
        "    \"\"\"Wrap a dataloader to move data to a device\"\"\"\n",
        "    def __init__(self, dl, device):\n",
        "        self.dl = dl\n",
        "        self.device = device\n",
        "        \n",
        "    def __iter__(self):\n",
        "        \"\"\"Yield a batch of data after moving it to device\"\"\"\n",
        "        for b in self.dl: \n",
        "            yield to_device(b, self.device)\n",
        "\n",
        "    def __len__(self):\n",
        "        \"\"\"Number of batches\"\"\"\n",
        "        return len(self.dl)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "id": "NKqCLHlA6Ksj"
      },
      "outputs": [],
      "source": [
        "max_length=200"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "id": "pETxnBgn50Za"
      },
      "outputs": [],
      "source": [
        "train_ds=JigsawDataset(train_df,tokenizer,max_length)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Xqxk51Y59io",
        "outputId": "f37a2486-4709-43e1-98f0-502584bcd724"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "({'input_ids': tensor([[  101, 10151, 14163, 16089,  1045,  1005,  2310,  2580,  1037, 24646,\n",
              "           2497,  2006, 10151, 14163, 16089,  2012, 10151, 14163, 16089,  2152,\n",
              "           2082,  1012,  2069,  2518,  1045,  2113,  2055,  2009,  2003,  2008,\n",
              "           2119,  2026,  5916, 10972,  1998,  3960, 24665,  3111,  2063,  2253,\n",
              "           2045,  1012,  3531,  5587,  2505,  2017,  2453,  2113,  2055,  2009,\n",
              "           1012, 18411,  2860,  1010,  2026,  3611,  2001,  1037, 15133,  1025,\n",
              "           1045,  2444,  1999,  9173,  2870,  1012,   102,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "          0, 0, 0, 0, 0, 0, 0, 0]])}, tensor([0., 0., 0., 0., 0., 0.]))"
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_ds[1000]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5CO1cZiGOAun"
      },
      "source": [
        "* BERT and other transformer-based models have their own embedding layer built in, which learns contextualized representations of the input tokens during pre-training. Therefore, there is no need to perform a separate embedding step before passing the input to the model.\n",
        "\n",
        "* Instead, you can simply tokenize the input text using a tokenizer, which will convert the input text into a sequence of numerical tokens that can be passed directly to the model for processing. The tokenizer may also add special tokens, such as [CLS] and [SEP], which are used to indicate the start and end of a sequence, respectively.\n",
        "\n",
        "* The [:, 0, :] indexing is used to extract the hidden state for the first token in the input sequence, which is a special token that is added by the DistilBERT tokenizer. Specifically, the first token corresponds to the [CLS] token, which is a special classification token that is used to represent the input sequence as a whole. By taking the hidden state of the [CLS] token, we can obtain a summary of the input sequence that can be used for classification purposes\n",
        "\n",
        "* e [CLS] token in the input sequence of a pre-trained language model like DistilBERT serves as a special token that is used to represent the input sequence as a whole. During training, the model learns to encode the input sequence in the hidden representation of the [CLS] token in a way that is useful for downstream tasks, such as text classification or sequence labeling.\n",
        "\n",
        "* By taking the hidden state of the [CLS] token, we can obtain a fixed-size representation of the input sequence that summarizes the most important information for the downstream task. This [CLS] token representation can then be fed to a classification layer or another downstream task-specific layer for further processing."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "VSFJPZG_7oct"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import random_split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "qv6Nsd607Qx5"
      },
      "outputs": [],
      "source": [
        "val_size=0.15\n",
        "x_train,x_val=random_split(train_ds,[1-val_size,val_size])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "id": "XNnnFSeg-ruC"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "id": "jfXXCEcW-vcp"
      },
      "outputs": [],
      "source": [
        "batch_size=32\n",
        "train_dl=DataLoader(x_train,batch_size,shuffle=True,pin_memory=True,num_workers=3)\n",
        "val_dl=DataLoader(x_val,batch_size*2,pin_memory=True,num_workers=3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LMWHj6Ymx0bF",
        "outputId": "4ba8f464-ac14-4a79-e73e-b4db6f198875"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([32, 200])\n",
            "torch.Size([32, 200])\n",
            "torch.Size([32, 6])\n"
          ]
        }
      ],
      "source": [
        "for batch in train_dl:\n",
        "  (inputs,target)=batch\n",
        "  id=inputs['input_ids']\n",
        "  mask=inputs['attention_mask']\n",
        "  id=id.squeeze(1)\n",
        "  mask=mask.squeeze(1)\n",
        "  print(id.shape) \n",
        "  print(mask.shape)\n",
        "  print(target.shape)\n",
        "  break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "4GJ9-UrMC4XF"
      },
      "outputs": [],
      "source": [
        "# train_dl=DeviceDataLoader(train_dl,device)\n",
        "# val_dl=DeviceDataLoader(val_dl,device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DFuTEL6ey58p",
        "outputId": "768bb139-26f5-43ca-e5ef-ff59faa6e91e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(4239, 374)"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_dl),len(val_dl)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EH_-ehMSxOhB"
      },
      "outputs": [],
      "source": [
        "for batch in train_dl:\n",
        "  print(batch)\n",
        "  break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "id": "gjG7P6EByxIQ"
      },
      "outputs": [],
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {
        "id": "WrahpAnDDDOM"
      },
      "outputs": [],
      "source": [
        "class DistilClass(nn.Module):\n",
        "  def __init__(self,num_classes=6):\n",
        "    super().__init__()\n",
        "    self.num_classes=num_classes\n",
        "    self.distbert=dis=DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
        "    self.pre_classifier=nn.Linear(768,768)\n",
        "    self.classifier=nn.Linear(768,self.num_classes)\n",
        "    self.dropout=nn.Dropout(0.2)\n",
        "    self.relu=nn.Tanh()\n",
        "  \n",
        "  def forward(self,input_ids,attention_mask):\n",
        "    distilbert_out=self.distbert(input_ids,attention_mask=attention_mask)\n",
        "    # print(distilbert_out.shape)\n",
        "    hidden_state=distilbert_out[0][:,0,:].to(device)\n",
        "    pooled_output=self.pre_classifier(hidden_state).to(device)\n",
        "    pooled_output=self.relu(pooled_output).to(device)\n",
        "    pooled_output=self.dropout(pooled_output).to(device)\n",
        "    outputs=self.classifier(pooled_output).to(device)\n",
        "    return outputs\n",
        "\n",
        "  def training_step(self,xb):\n",
        "    inputs,targets=xb\n",
        "    id=inputs['input_ids'].to(device)\n",
        "    mask=inputs['attention_mask'].to(device)\n",
        "    targets=targets.to(device)\n",
        "    id=id.squeeze(1)\n",
        "    mask=mask.squeeze(1)\n",
        "    out=self(id,mask)\n",
        "    # out=torch.sigmoid(out)\n",
        "    loss=torch.nn.BCEWithLogitsLoss()(out,targets).to(device)\n",
        "    return loss\n",
        "  \n",
        "  def validation_step(self,xb):\n",
        "    inputs,targets=xb\n",
        "    id=inputs['input_ids'].to(device)\n",
        "    mask=inputs['attention_mask'].to(device)\n",
        "    targets=targets.to(device)\n",
        "    id=id.squeeze(1)\n",
        "    mask=mask.squeeze(1)\n",
        "    out=self(id,mask)\n",
        "    # out=torch.sigmoid(out)\n",
        "    loss=nn.BCEWithLogitsLoss()(out,targets).to(device)\n",
        "    return {'val_loss':loss}\n",
        "  \n",
        "  def validation_epoch_end(self, outputs):  \n",
        "     batch_losses = [x['val_loss'] for x in outputs]\n",
        "     epoch_loss = torch.stack(batch_losses).mean()   # Combine losses\n",
        "     return {'val_loss': epoch_loss.item()}\n",
        "     \n",
        "  def epoch_end(self,epoch,result):\n",
        "    print(\"Epoch [{}] ,Train_Loss:{:.4f}, Val_Loss:{:.4f}\".format(epoch,result['train_loss'],result['val_loss']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zlyRQZoOF4et",
        "outputId": "df6b64f3-b72a-41a7-8408-a4af98fd14f8"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_transform.bias']\n",
            "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ],
      "source": [
        "model=DistilClass().to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {
        "id": "tqrf_EUlUu0r"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "metadata": {
        "id": "pL37Qnl5lSSx"
      },
      "outputs": [],
      "source": [
        "@torch.no_grad()\n",
        "def evaluate(model, val_loader):\n",
        "    model.eval()\n",
        "    outputs = [model.validation_step(batch) for batch in val_loader]\n",
        "    return model.validation_epoch_end(outputs)\n",
        "\n",
        "def fit(epochs,lr,model,train_dl,val_dl,opt_func=torch.optim.SGD):\n",
        "  optimizer = opt_func(model.parameters(), lr)\n",
        "  torch.cuda.empty_cache()\n",
        "  history=[]\n",
        "  for epoch in range(epochs):\n",
        "      model.train()\n",
        "      train_losses=[]\n",
        "      for batch in train_dl:\n",
        "        loss= model.training_step(batch)\n",
        "        train_losses.append(loss)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "      result = evaluate(model, val_dl)\n",
        "      result['train_loss']=torch.stack(train_losses).mean().item()\n",
        "      model.epoch_end(epoch, result)\n",
        "      history.append(result)\n",
        "  return history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "metadata": {
        "id": "Lkz-43EWKmoF"
      },
      "outputs": [],
      "source": [
        "num_epochs = 2\n",
        "opt_func = torch.optim.AdamW\n",
        "lr = 1e-4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 70,
      "metadata": {
        "id": "C4s9tf8uy0Yb"
      },
      "outputs": [],
      "source": [
        "history=evaluate(model,val_dl)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 71,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GQAakE-zfYGw",
        "outputId": "1946c908-b89d-4f79-a4c3-ee472ef8b48c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'val_loss': 0.6767199635505676}"
            ]
          },
          "execution_count": 71,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IsS26wn6Lo-a",
        "outputId": "67c76491-610f-4b73-a651-87d85602c904"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch [0] ,Train_Loss:0.0469, Val_Loss:0.0438\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch [1] ,Train_Loss:0.0378, Val_Loss:0.0468\n"
          ]
        }
      ],
      "source": [
        "history=fit(num_epochs,lr,model,train_dl,val_dl,opt_func)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 76,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zvQLJhPQjJDc",
        "outputId": "00fcace6-99de-4c97-c109-47c34b2a2436"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'val_loss': 0.043839823454618454, 'train_loss': 0.04693041741847992},\n",
              " {'val_loss': 0.04681534692645073, 'train_loss': 0.037766341120004654}]"
            ]
          },
          "execution_count": 76,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 77,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "K26U47pASd0n",
        "outputId": "07990c5a-7670-410f-dfcf-3c69c5c74546"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-9e64ac67-7a8f-4256-8669-1573aa382dc4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Yo bitch Ja Rule is more succesful then you'll...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>== From RfC == \\n\\n The title is fine as it is...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\" \\n\\n == Sources == \\n\\n * Zawe Ashton on Lap...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>:If you have a look back at the source, the in...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I don't anonymously edit articles at all.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153159</th>\n",
              "      <td>. \\n i totally agree, this stuff is nothing bu...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153160</th>\n",
              "      <td>== Throw from out field to home plate. == \\n\\n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153161</th>\n",
              "      <td>\" \\n\\n == Okinotorishima categories == \\n\\n I ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153162</th>\n",
              "      <td>\" \\n\\n == \"\"One of the founding nations of the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153163</th>\n",
              "      <td>\" \\n :::Stop already. Your bullshit is not wel...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>153164 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9e64ac67-7a8f-4256-8669-1573aa382dc4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9e64ac67-7a8f-4256-8669-1573aa382dc4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9e64ac67-7a8f-4256-8669-1573aa382dc4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                             comment_text\n",
              "0       Yo bitch Ja Rule is more succesful then you'll...\n",
              "1       == From RfC == \\n\\n The title is fine as it is...\n",
              "2       \" \\n\\n == Sources == \\n\\n * Zawe Ashton on Lap...\n",
              "3       :If you have a look back at the source, the in...\n",
              "4               I don't anonymously edit articles at all.\n",
              "...                                                   ...\n",
              "153159  . \\n i totally agree, this stuff is nothing bu...\n",
              "153160  == Throw from out field to home plate. == \\n\\n...\n",
              "153161  \" \\n\\n == Okinotorishima categories == \\n\\n I ...\n",
              "153162  \" \\n\\n == \"\"One of the founding nations of the...\n",
              "153163  \" \\n :::Stop already. Your bullshit is not wel...\n",
              "\n",
              "[153164 rows x 1 columns]"
            ]
          },
          "execution_count": 77,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_df=test_df.drop('id',axis=1)\n",
        "test_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 78,
      "metadata": {
        "id": "89M5qxbTSvN-"
      },
      "outputs": [],
      "source": [
        "test_ds=JigsawDataset(test_df,tokenizer,max_length,test=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 79,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TDdC3GpVQem-",
        "outputId": "97a710f8-57dc-4d65-a87a-b6ca72fb6b4a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "153164\n"
          ]
        }
      ],
      "source": [
        "print(len(test_ds))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 81,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "nnlYZ1rIWwfY",
        "outputId": "f8c63436-506a-4731-fcc8-0c4bff25cb2d"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"Yo bitch Ja Rule is more succesful then you'll ever be whats up with you and hating you sad mofuckas...i should bitch slap ur pethedic white faces and get you to kiss my ass you guys sicken me. Ja rule is about pride in da music man. dont diss that shit on him. and nothin is wrong bein like tupac he was a brother too...fuckin white boys get things right next time.,\""
            ]
          },
          "execution_count": 81,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_df.comment_text[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 83,
      "metadata": {
        "id": "zq_a2jsqaIFl"
      },
      "outputs": [],
      "source": [
        "preds=[]\n",
        "def predict(model,test_ds):\n",
        "  with torch.no_grad():\n",
        "    for batch in test_ds:\n",
        "      inputs,targets=batch\n",
        "      id=inputs['input_ids'].to(device)\n",
        "      mask=inputs['attention_mask'].to(device)\n",
        "      id=id.squeeze(1)\n",
        "      mask=mask.squeeze(1)\n",
        "      out=model(id,mask)\n",
        "      output=torch.sigmoid(out)\n",
        "      preds.append(output.squeeze(0))\n",
        "predict(model,test_ds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 85,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "somXonlnyRUk",
        "outputId": "10ed9d13-3a26-4b77-ef2b-3d52a8ab7698"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "153164"
            ]
          },
          "execution_count": 85,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(preds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 86,
      "metadata": {
        "id": "jrp869a5W3uS"
      },
      "outputs": [],
      "source": [
        "for i in range(len(preds)):\n",
        "  preds[i]=(preds[i]>0.55).int()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 87,
      "metadata": {
        "id": "K6WBjZfFeTvV"
      },
      "outputs": [],
      "source": [
        "for i in range(len(preds)):\n",
        "  preds[i]=preds[i].detach().cpu().numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 88,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "llSOJ6HvgPTm",
        "outputId": "1296ee4e-7ff3-47b4-a124-c184db5078de"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "153164"
            ]
          },
          "execution_count": 88,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(preds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rtq2nfaz3zYU"
      },
      "outputs": [],
      "source": [
        "preds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 90,
      "metadata": {
        "id": "zF9ujz5CXYR1"
      },
      "outputs": [],
      "source": [
        "sub_df[cols]=preds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 91,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "6-oL0wjhhnho",
        "outputId": "2fba9784-c263-4fc7-9fcf-3b1d0f320445"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-db61a63b-c723-4698-a40d-0bef5633f228\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00001cee341fdb12</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0000247867823ef7</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>00013b17ad220c46</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>00017563c3f7919a</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>00017695ad8997eb</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153159</th>\n",
              "      <td>fffcd0960ee309b5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153160</th>\n",
              "      <td>fffd7a9a6eb32c16</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153161</th>\n",
              "      <td>fffda9e8d6fafa9e</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153162</th>\n",
              "      <td>fffe8f1340a79fc2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153163</th>\n",
              "      <td>ffffce3fb183ee80</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>153164 rows × 7 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-db61a63b-c723-4698-a40d-0bef5633f228')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-db61a63b-c723-4698-a40d-0bef5633f228 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-db61a63b-c723-4698-a40d-0bef5633f228');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                      id  toxic  severe_toxic  obscene  threat  insult  \\\n",
              "0       00001cee341fdb12      1             0        1       0       1   \n",
              "1       0000247867823ef7      0             0        0       0       0   \n",
              "2       00013b17ad220c46      0             0        0       0       0   \n",
              "3       00017563c3f7919a      0             0        0       0       0   \n",
              "4       00017695ad8997eb      0             0        0       0       0   \n",
              "...                  ...    ...           ...      ...     ...     ...   \n",
              "153159  fffcd0960ee309b5      0             0        0       0       0   \n",
              "153160  fffd7a9a6eb32c16      0             0        0       0       0   \n",
              "153161  fffda9e8d6fafa9e      0             0        0       0       0   \n",
              "153162  fffe8f1340a79fc2      0             0        0       0       0   \n",
              "153163  ffffce3fb183ee80      1             0        1       0       0   \n",
              "\n",
              "        identity_hate  \n",
              "0                   1  \n",
              "1                   0  \n",
              "2                   0  \n",
              "3                   0  \n",
              "4                   0  \n",
              "...               ...  \n",
              "153159              0  \n",
              "153160              0  \n",
              "153161              0  \n",
              "153162              0  \n",
              "153163              0  \n",
              "\n",
              "[153164 rows x 7 columns]"
            ]
          },
          "execution_count": 91,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sub_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 92,
      "metadata": {
        "id": "EsJ6k6yF4ALJ"
      },
      "outputs": [],
      "source": [
        "sub_df.to_csv('submission.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 95,
      "metadata": {
        "id": "WQXdlPaIhwLB"
      },
      "outputs": [],
      "source": [
        "@torch.no_grad()\n",
        "def make_prediction(model,sentence):\n",
        "  inputs=tokenizer.encode_plus(sentence)\n",
        "  id=torch.tensor(inputs['input_ids'],dtype=torch.int).to(device)\n",
        "  mask=torch.tensor(inputs['attention_mask'],dtype=torch.float).to(device)\n",
        "  id=id.unsqueeze(0)\n",
        "  mask=mask.unsqueeze(0)\n",
        "  out=model(id,mask)\n",
        "  output=torch.sigmoid(out)\n",
        "  output=(output[0]>0.55).int()\n",
        "  output=output.detach().cpu().numpy()\n",
        "  return output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 104,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DUZLv5qxkhs4",
        "outputId": "b599ee12-5d9c-44c5-a75f-775f9a874491"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([1, 0, 1, 0, 1, 0], dtype=int32)"
            ]
          },
          "execution_count": 104,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sentence=\"Maybe you should have a read of that one too. Since I've decided to assist with articles on wikipedia, you have made the experience nothing but hell you piece of shit. Pull your finger out of your bureaucratic arse and rot in hell.\"\n",
        "out=make_prediction(model,sentence)\n",
        "out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N6U8RZ06l3J2",
        "outputId": "f222bf41-3aa8-4652-d09b-9e238c94539a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']"
            ]
          },
          "execution_count": 99,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cols"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 80,
      "metadata": {
        "id": "JMKNvqComc2Y"
      },
      "outputs": [],
      "source": [
        "torch.save(model.state_dict(),'model.pt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MbE_vPoHyEfK"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.10"
    },
    "vscode": {
      "interpreter": {
        "hash": "26de051ba29f2982a8de78e945f0abaf191376122a1563185a90213a26c5da77"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "07efa77917f44b9a8f7416ee4bc1ca0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10a4b3ac62a94f01942fd3ce8af68cfc",
            "placeholder": "​",
            "style": "IPY_MODEL_645c94627ff542119525b77c9ad172fb",
            "value": " 232k/232k [00:00&lt;00:00, 261kB/s]"
          }
        },
        "0b6ad200b4924ebda9509fa3be12caa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ea453dc7eec409e86d119fb5d40ac2e",
            "placeholder": "​",
            "style": "IPY_MODEL_3a71d2dc01a4473fbc71a7fdf1cb2afe",
            "value": " 28.0/28.0 [00:00&lt;00:00, 953B/s]"
          }
        },
        "10a4b3ac62a94f01942fd3ce8af68cfc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "11a9fb67f1e64443a0b679440483c947": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1328d06b169c43c7a5d0a9ea5a1bcf8c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21478fea93704608b18523074e9c81bf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2262ba8c53be48aebed1f162791ef9f6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "273ad16d3ff34175abe8b249951ddfe6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3a71d2dc01a4473fbc71a7fdf1cb2afe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3b4f9a31690f4513bc557c31113cb3fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5a965e00ff5146539b930ee7af62f6b3",
              "IPY_MODEL_4f3b42b77ebd4cf4bc11f88653801e82",
              "IPY_MODEL_0b6ad200b4924ebda9509fa3be12caa0"
            ],
            "layout": "IPY_MODEL_c056b40910a14abab93beac0d8f5fef8"
          }
        },
        "4f3b42b77ebd4cf4bc11f88653801e82": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_59cbef76baf64a5183377873d0dd4d05",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_11a9fb67f1e64443a0b679440483c947",
            "value": 28
          }
        },
        "58f1754a3f954dc9a582bec5d63baae5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59cbef76baf64a5183377873d0dd4d05": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a965e00ff5146539b930ee7af62f6b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8f4c51eb03647eabc125ad6f19bdc8f",
            "placeholder": "​",
            "style": "IPY_MODEL_be25c816264147f3a774d1cd977a6ddb",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "645c94627ff542119525b77c9ad172fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "68f7235619834d9d9f1d164a63319824": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2262ba8c53be48aebed1f162791ef9f6",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dabb0d270e5a4e088a977f3a144c3a18",
            "value": 231508
          }
        },
        "6ea453dc7eec409e86d119fb5d40ac2e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ba1b98098ee49abb81c64ba863f8805": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91c3e20384384fc1aa57b21c64a1909a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a7ccc548d0444837a21cfcd9b9474a42",
              "IPY_MODEL_68f7235619834d9d9f1d164a63319824",
              "IPY_MODEL_07efa77917f44b9a8f7416ee4bc1ca0e"
            ],
            "layout": "IPY_MODEL_58f1754a3f954dc9a582bec5d63baae5"
          }
        },
        "a520f5c368884575a7b9be38f0b0b3e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a7ccc548d0444837a21cfcd9b9474a42": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21478fea93704608b18523074e9c81bf",
            "placeholder": "​",
            "style": "IPY_MODEL_e07506c478a846f1ab24591953b5a312",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "b7c7dad6dcb04d6fba4b22f646f6ed78": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_de30acd76ffe406b8f63d2f26a86bd6f",
            "placeholder": "​",
            "style": "IPY_MODEL_c15beb3e41d54972a30f8cfd2d21ad72",
            "value": " 483/483 [00:00&lt;00:00, 17.5kB/s]"
          }
        },
        "be25c816264147f3a774d1cd977a6ddb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c056b40910a14abab93beac0d8f5fef8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c15beb3e41d54972a30f8cfd2d21ad72": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c55e15328e3444a88b44fb00cf19cd60": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1328d06b169c43c7a5d0a9ea5a1bcf8c",
            "max": 483,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_273ad16d3ff34175abe8b249951ddfe6",
            "value": 483
          }
        },
        "dabb0d270e5a4e088a977f3a144c3a18": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dcfd001961204b1685f4657205cb50e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fee9288216d1422e96b54b140578ec14",
              "IPY_MODEL_c55e15328e3444a88b44fb00cf19cd60",
              "IPY_MODEL_b7c7dad6dcb04d6fba4b22f646f6ed78"
            ],
            "layout": "IPY_MODEL_f743507a96194537aec2a3fcda764c85"
          }
        },
        "de30acd76ffe406b8f63d2f26a86bd6f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e07506c478a846f1ab24591953b5a312": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f743507a96194537aec2a3fcda764c85": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f8f4c51eb03647eabc125ad6f19bdc8f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fee9288216d1422e96b54b140578ec14": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8ba1b98098ee49abb81c64ba863f8805",
            "placeholder": "​",
            "style": "IPY_MODEL_a520f5c368884575a7b9be38f0b0b3e3",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}