3-Template-RoBERTa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun Nov 27 14:06:16 2022       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 520.61.05    Driver Version: 520.61.05    CUDA Version: 11.8     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA GeForce ...  Off  | 00000000:01:00.0 Off |                  N/A |\n",
      "|  0%   44C    P8    14W / 170W |      0MiB / 12288MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gensim\n",
    "import sklearn.metrics\n",
    "import re\n",
    "import unicodedata\n",
    "from tqdm import tqdm\n",
    "import torch\n",
    "from simpletransformers.classification import ClassificationModel, ClassificationArgs\n",
    "import wandb\n",
    "import csv\n",
    "from functools import partial\n",
    "import itertools\n",
    "import random\n",
    "import sys\n",
    "import hashlib\n",
    "import time\n",
    "\n",
    "train_file = 'nlbse23-issue-classification-train.csv'\n",
    "test_file = 'nlbse23-issue-classification-test.csv'\n",
    "\n",
    "csv.field_size_limit(sys.maxsize) # to avoid error: _csv.Error: field larger than field limit (131072)\n",
    "\n",
    "def count_tokens(text):\n",
    "\treturn len(text.split())\n",
    "\n",
    "def count_csv_rows(csv_file):\n",
    "\twith open(csv_file, 'r', newline='', encoding='utf-8') as f:\n",
    "\t\treturn sum(1 for _ in csv.DictReader(f))\n",
    "\n",
    "def print_csv_preview(filename, sep=None):\n",
    "\tprint(filename)\n",
    "\tprint(\"total rows\", count_csv_rows(filename))\n",
    "\tdisplay(pd.read_csv(filename, nrows=5, sep=sep))\n",
    "\n",
    "def sample_csv(file, n_sample):\n",
    "\tn_population = count_csv_rows(file)\n",
    "\tskiprows = random.sample(range(1, n_population), n_population - n_sample)\n",
    "\treturn pd.read_csv(file, skiprows=skiprows)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nlbse23-issue-classification-train.csv\n",
      "total rows 1275881\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>labels</th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>author_association</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1199051804</td>\n",
       "      <td>documentation</td>\n",
       "      <td>setting a logging Handler name</td>\n",
       "      <td>BPO | [43058](https://bugs.python.org/issue430...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1199074324</td>\n",
       "      <td>documentation</td>\n",
       "      <td>Improve documentation for typing._GenericAlias</td>\n",
       "      <td>BPO | [46589](https://bugs.python.org/issue465...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1199022454</td>\n",
       "      <td>documentation</td>\n",
       "      <td>Description of '\\w' behavior is vague in `re` ...</td>\n",
       "      <td>BPO | [38566](https://bugs.python.org/issue385...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1199028356</td>\n",
       "      <td>documentation</td>\n",
       "      <td>add docstrings to functions in pdb module</td>\n",
       "      <td>BPO | [39278](https://bugs.python.org/issue392...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1199055394</td>\n",
       "      <td>documentation</td>\n",
       "      <td>Documentation needs to declare CalledProcessEr...</td>\n",
       "      <td>BPO | [43635](https://bugs.python.org/issue436...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id         labels  \\\n",
       "0  1199051804  documentation   \n",
       "1  1199074324  documentation   \n",
       "2  1199022454  documentation   \n",
       "3  1199028356  documentation   \n",
       "4  1199055394  documentation   \n",
       "\n",
       "                                               title  \\\n",
       "0                     setting a logging Handler name   \n",
       "1     Improve documentation for typing._GenericAlias   \n",
       "2  Description of '\\w' behavior is vague in `re` ...   \n",
       "3          add docstrings to functions in pdb module   \n",
       "4  Documentation needs to declare CalledProcessEr...   \n",
       "\n",
       "                                                body author_association  \n",
       "0  BPO | [43058](https://bugs.python.org/issue430...          MANNEQUIN  \n",
       "1  BPO | [46589](https://bugs.python.org/issue465...          MANNEQUIN  \n",
       "2  BPO | [38566](https://bugs.python.org/issue385...          MANNEQUIN  \n",
       "3  BPO | [39278](https://bugs.python.org/issue392...          MANNEQUIN  \n",
       "4  BPO | [43635](https://bugs.python.org/issue436...          MANNEQUIN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nlbse23-issue-classification-test.csv\n",
      "total rows 142320\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>labels</th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>author_association</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1199053386</td>\n",
       "      <td>documentation</td>\n",
       "      <td>A possible misleading expression in the Virtua...</td>\n",
       "      <td>BPO | [43319](https://bugs.python.org/issue433...</td>\n",
       "      <td>MANNEQUIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1255069635</td>\n",
       "      <td>bug</td>\n",
       "      <td>[BUG] a valid `gameName` in the `create a new ...</td>\n",
       "      <td>**Describe the bug**\\r\\nIn the `create a new p...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1089772715</td>\n",
       "      <td>feature</td>\n",
       "      <td>How to check if a certain entity still exists?</td>\n",
       "      <td>During a bug in my own code I noticed that the...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000928729</td>\n",
       "      <td>feature</td>\n",
       "      <td>chose the timezone in dbeaver option</td>\n",
       "      <td>Dbeaver 21.2.0\\r\\n\\r\\nFor all version DBeaver,...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1300011093</td>\n",
       "      <td>bug</td>\n",
       "      <td>[Issue]: Multiple Versions of a Movie not work...</td>\n",
       "      <td>### Please describe your bug\\n\\nThe doc at htt...</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id         labels  \\\n",
       "0  1199053386  documentation   \n",
       "1  1255069635            bug   \n",
       "2  1089772715        feature   \n",
       "3  1000928729        feature   \n",
       "4  1300011093            bug   \n",
       "\n",
       "                                               title  \\\n",
       "0  A possible misleading expression in the Virtua...   \n",
       "1  [BUG] a valid `gameName` in the `create a new ...   \n",
       "2     How to check if a certain entity still exists?   \n",
       "3               chose the timezone in dbeaver option   \n",
       "4  [Issue]: Multiple Versions of a Movie not work...   \n",
       "\n",
       "                                                body author_association  \n",
       "0  BPO | [43319](https://bugs.python.org/issue433...          MANNEQUIN  \n",
       "1  **Describe the bug**\\r\\nIn the `create a new p...               NONE  \n",
       "2  During a bug in my own code I noticed that the...               NONE  \n",
       "3  Dbeaver 21.2.0\\r\\n\\r\\nFor all version DBeaver,...               NONE  \n",
       "4  ### Please describe your bug\\n\\nThe doc at htt...               NONE  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# download the training set if it does not exist\n",
    "if not os.path.isfile(train_file):\n",
    "  !curl \"https://tickettagger.blob.core.windows.net/datasets/{train_file}.tar.gz\" | tar -xz\n",
    "\n",
    "print_csv_preview(train_file)\n",
    "\n",
    "if not os.path.isfile(test_file):\n",
    "  !curl \"https://tickettagger.blob.core.windows.net/datasets/{test_file}.tar.gz\" | tar -xz\n",
    "\n",
    "print_csv_preview(test_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "function_sig_regex = re.compile(r'[a-zA-Z][a-zA-Z0-9_.]*\\([a-zA-Z0-9_, ]*\\)')\n",
    "issue_id_regex = re.compile(r'#[0-9]+')\n",
    "non_ascii_char_regex = re.compile(r'[^\\x00-\\x7f]')\n",
    "punctuations = '!\\'\"`$%&\\()*,/:;<=>[\\\\]^{|}~+#@-_'\n",
    "punctuations_trans = str.maketrans(punctuations, \" \" * len(punctuations))\n",
    "\n",
    "def preprocess(text, max_tokens=None):\n",
    "  text = str(text)\n",
    "\n",
    "  # replace function signatures\n",
    "  text = function_sig_regex.sub(\" FUNCTION \", text)\n",
    "\n",
    "  # replace issue ids\n",
    "  text = issue_id_regex.sub(\" ISSUE \", text)\n",
    "  \n",
    "  # remove html tags\n",
    "  # text = gensim.parsing.preprocessing.strip_tags(text)\n",
    "  \n",
    "  # remove punctuation\n",
    "  text = text.translate(punctuations_trans)\n",
    "  \n",
    "  # remove numerics\n",
    "  # text = gensim.parsing.preprocessing.strip_numeric(text)\n",
    "  \n",
    "  # remove non-ascii characters\n",
    "  text = non_ascii_char_regex.sub(\"\", text)\n",
    "  \n",
    "  text = unicodedata.normalize('NFD', text)\n",
    "  \n",
    "  # remove consecutive whitespace characters and convert tabs to spaces\n",
    "  text = gensim.parsing.preprocessing.strip_multiple_whitespaces(text)\n",
    "  \n",
    "  # limit the number of tokens\n",
    "  if max_tokens is not None:\n",
    "    text = \" \".join(text.split()[:max_tokens])\n",
    "  \n",
    "  return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'title token count quantiles'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.500     7.0\n",
       "0.750    10.0\n",
       "0.800    10.0\n",
       "0.850    11.0\n",
       "0.900    12.0\n",
       "0.950    15.0\n",
       "0.990    20.0\n",
       "0.999    30.0\n",
       "Name: title, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'body token count quantiles'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.500      72.000\n",
       "0.750     152.000\n",
       "0.800     181.000\n",
       "0.850     223.000\n",
       "0.900     296.000\n",
       "0.950     464.000\n",
       "0.990    1414.030\n",
       "0.999    6433.146\n",
       "Name: body, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample_df = sample_csv(train_file, 50_000)\n",
    "\n",
    "q=[.5, .75, .8, .85, .9, .95, .99, .999]\n",
    "\n",
    "display(\"title token count quantiles\", sample_df[\"title\"].apply(preprocess).apply(count_tokens).quantile(q=q))\n",
    "display(\"body token count quantiles\", sample_df[\"body\"].apply(preprocess).apply(count_tokens).quantile(q=q))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05b7e48b372242288e44dc9dae3206b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Transform to simpletransformers format:   0%|          | 0/1275881 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af2fcd516c794659af78b3010124113b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Transform to simpletransformers format:   0%|          | 0/142320 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train.csv\n",
      "total rows 1275881\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TITLE setting a logging Handler name BODY BPO ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TITLE Improve documentation for typing. Generi...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TITLE Description of w behavior is vague in re...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TITLE add docstrings to functions in pdb modul...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TITLE Documentation needs to declare CalledPro...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  labels\n",
       "0  TITLE setting a logging Handler name BODY BPO ...       3\n",
       "1  TITLE Improve documentation for typing. Generi...       3\n",
       "2  TITLE Description of w behavior is vague in re...       3\n",
       "3  TITLE add docstrings to functions in pdb modul...       3\n",
       "4  TITLE Documentation needs to declare CalledPro...       3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test.csv\n",
      "total rows 142320\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TITLE A possible misleading expression in the ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TITLE BUG a valid gameName in the create a new...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TITLE How to check if a certain entity still e...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TITLE chose the timezone in dbeaver option BOD...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TITLE Issue Multiple Versions of a Movie not w...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  labels\n",
       "0  TITLE A possible misleading expression in the ...       3\n",
       "1  TITLE BUG a valid gameName in the create a new...       0\n",
       "2  TITLE How to check if a certain entity still e...       1\n",
       "3  TITLE chose the timezone in dbeaver option BOD...       1\n",
       "4  TITLE Issue Multiple Versions of a Movie not w...       0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# transform dataset into simpletransformers format\n",
    "# https://simpletransformers.ai/docs/classification-data-formats/#multi-class-classification\n",
    "\n",
    "def preprocess_row(row):\n",
    "  doc = \"TITLE \" + preprocess(row[\"title\"], max_tokens=20) # 99% of titles fit\n",
    "  doc += \" BODY \" + preprocess(row[\"body\"], max_tokens=511-count_tokens(doc))\n",
    "\n",
    "  assert count_tokens(doc) <= 512\n",
    "\n",
    "  return doc\n",
    "\n",
    "def transform_to_simpletransformers_format(i_path, o_path):\n",
    "\tlabel_map = {\"bug\": 0, \"feature\": 1, \"question\": 2, \"documentation\": 3 }\n",
    "\n",
    "\twith open(i_path, \"r\") as i_f, open(o_path, \"w\") as o_f:\n",
    "\t\treader = csv.DictReader(i_f)\n",
    "\t\twriter = csv.DictWriter(o_f, fieldnames=[\"text\", \"labels\"], delimiter=\"\\t\")\n",
    "\t\twriter.writeheader()\n",
    "\t\ttotal = count_csv_rows(i_path)\n",
    "\t\tfor row in tqdm(reader, desc=\"Transform to simpletransformers format\", total=total):\n",
    "\t\t\ttext = preprocess_row(row)\n",
    "\t\t\tlabels = label_map[row[\"labels\"]]\n",
    "\t\t\twriter.writerow({\"text\": text, \"labels\": labels})\n",
    "\n",
    "transform_to_simpletransformers_format(train_file, \"train.csv\")\n",
    "transform_to_simpletransformers_format(test_file, \"test.csv\")\n",
    "\n",
    "print_csv_preview(\"train.csv\", sep='\\t')\n",
    "print_csv_preview(\"test.csv\", sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def model_args():\n",
    "  timestamp = str(int(time.time()))\n",
    "\n",
    "  # https://simpletransformers.ai/docs/usage/#configuring-a-simple-transformers-model\n",
    "  args = ClassificationArgs()\n",
    "\n",
    "  args.max_seq_length = 224\n",
    "  args.learning_rate = 1e-5 # 4e-5\n",
    "  args.num_train_epochs = 16\n",
    "  args.train_batch_size = 64\n",
    "  args.eval_batch_size = 64\n",
    "  args.gradient_accumulation_steps = 1\n",
    "\n",
    "  # custom evaluation metric\n",
    "  # https://github.com/ThilinaRajapakse/simpletransformers/discussions/911\n",
    "  args.use_early_stopping = False\n",
    "  args.early_stopping_metric = \"f1_micro\"\n",
    "  args.early_stopping_metric_minimize = False\n",
    "\n",
    "  # evaluate at end of each epoch\n",
    "  args.evaluate_during_training = True\n",
    "  args.evaluate_during_training_steps = int(1e20) # never\n",
    "  # args.evaluate_during_training_steps = 1.5  * 60 * 60 // args.gradient_accumulation_steps # target 1h\n",
    "\n",
    "  # https://simpletransformers.ai/docs/classification-specifics/#lazy-loading-data\n",
    "  args.lazy_loading = True\n",
    "\n",
    "  args.save_steps = -1\n",
    "  args.logging_steps = max(1, 1.6 * 30 // args.gradient_accumulation_steps) # target 30s\n",
    "  args.manual_seed = 0\n",
    "\n",
    "  args.output_dir = f\"outputs/{timestamp}\"\n",
    "  args.best_model_dir = f\"{args.output_dir}/best_model\"\n",
    "\n",
    "  # https://docs.wandb.ai/guides/integrations/other/simpletransformers\n",
    "  # https://simpletransformers.ai/docs/tips-and-tricks/#visualization-support\n",
    "  args.wandb_project = \"NLBSE'23 Issue Report Classification\"\n",
    "  args.wandb_kwargs = {\"entity\": \"nlbse\", \"notes\": f\"timestamp:{timestamp}\"}\n",
    "\n",
    "  return args\n",
    "\n",
    "metrics = {\n",
    "  \"precision_bug\": partial(sklearn.metrics.precision_score, average=None, labels=[0]),\n",
    "  \"recall_bug\": partial(sklearn.metrics.recall_score, average=None, labels=[0]),\n",
    "  \"f1_bug\": partial(sklearn.metrics.f1_score, average=None, labels=[0]),\n",
    "\n",
    "  \"precision_feature\": partial(sklearn.metrics.precision_score, average=None, labels=[1]),\n",
    "  \"recall_feature\": partial(sklearn.metrics.recall_score, average=None, labels=[1]),\n",
    "  \"f1_feature\": partial(sklearn.metrics.f1_score, average=None, labels=[1]),\n",
    "\n",
    "  \"precision_question\": partial(sklearn.metrics.precision_score, average=None, labels=[2]),\n",
    "  \"recall_question\": partial(sklearn.metrics.recall_score, average=None, labels=[2]),\n",
    "  \"f1_question\": partial(sklearn.metrics.f1_score, average=None, labels=[2]),\n",
    "\n",
    "  \"precision_documentation\": partial(sklearn.metrics.precision_score, average=None, labels=[3]),\n",
    "  \"recall_documentation\": partial(sklearn.metrics.recall_score, average=None, labels=[3]),\n",
    "  \"f1_documentation\": partial(sklearn.metrics.f1_score, average=None, labels=[3]),\n",
    "\n",
    "  \"precision_micro\": partial(sklearn.metrics.precision_score, average='micro'),\n",
    "  \"recall_micro\": partial(sklearn.metrics.recall_score, average='micro'),\n",
    "  \"f1_micro\": partial(sklearn.metrics.f1_score, average='micro'),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ClassificationModel(\n",
    "  'roberta', \n",
    "  'roberta-base', \n",
    "  args=model_args(), \n",
    "  num_labels=4\n",
    ")\n",
    "\n",
    "model.train_model(train_df=\"train.csv\", eval_df=\"test.csv\", **metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d4186e8b0cf4b80b13f97ee65928e01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Evaluation:   0%|          | 0/2224 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "/home/rafaelkallis/.local/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2323: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n",
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mrafaelkallis\u001b[0m (\u001b[33mnlbse\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.13.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/rafaelkallis/Notebooks/wandb/run-20221127_141450-39qkmfyf</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/nlbse/NLBSE%2723%20Issue%20Report%20Classification/runs/39qkmfyf\" target=\"_blank\">fragrant-serenity-28</a></strong> to <a href=\"https://wandb.ai/nlbse/NLBSE%2723%20Issue%20Report%20Classification\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m wandb.plots.* functions are deprecated and will be removed in a future release. Please use wandb.plot.* instead.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m wandb uses only the first 10000 datapoints to create the plots.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m wandb uses only the first 10000 datapoints to create the plots.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m wandb uses only the first 10000 datapoints to create the plots.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m wandb uses only the first 10000 datapoints to create the plots.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mcc': 0.8095082143241071,\n",
       " 'precision_bug': array([0.9110252]),\n",
       " 'recall_bug': array([0.93900857]),\n",
       " 'f1_bug': array([0.92480525]),\n",
       " 'precision_feature': array([0.89504329]),\n",
       " 'recall_feature': array([0.8967555]),\n",
       " 'f1_feature': array([0.89589858]),\n",
       " 'precision_question': array([0.73099061]),\n",
       " 'recall_question': array([0.56843345]),\n",
       " 'f1_question': array([0.63954413]),\n",
       " 'precision_documentation': array([0.75949147]),\n",
       " 'recall_documentation': array([0.69753679]),\n",
       " 'f1_documentation': array([0.72719693]),\n",
       " 'precision_micro': 0.8906197301854974,\n",
       " 'recall_micro': 0.8906197301854974,\n",
       " 'f1_micro': 0.8906197301854974,\n",
       " 'eval_loss': 0.3329971030274098}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = model = ClassificationModel(\n",
    "\t\"roberta\",\n",
    "\t\"outputs/best_model\",\n",
    "  args=model_args(), \n",
    "  num_labels=4,\n",
    ")\n",
    "\n",
    "results, model_outputs, wrong_pred = model.eval_model(eval_df=\"test.csv\", **metrics)\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation\n",
    "\n",
    "In this section we evaluate our trained model.\n",
    "We uploaded our trained model to the HuggingFace Hub. You can find it [here](https://huggingface.co/rafaelkallis/tickettagger-roberta)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bc8868e4e654d28badaff7dc8ca8723",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.19k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from transformers import pipeline\n",
    "from datasets import load_dataset\n",
    "from transformers.pipelines.pt_utils import KeyDataset\n",
    "import sklearn.metrics\n",
    "\n",
    "# classifier = pipeline(model=\"./best_model/\", tokenizer=\"./best_model/\")\n",
    "classifier = pipeline(model=\"rafaelkallis/tickettagger-roberta\", max_length=224, truncation=True, device=0, batch_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-04b119ce8cd31270\n",
      "Found cached dataset csv (/home/rafaelkallis/.cache/huggingface/datasets/csv/default-04b119ce8cd31270/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a1b85eeee05491cbe90e1e5c58ea662",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "142320it [18:29, 128.28it/s]                 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=*= bug =*=\n",
      "precision:\t0.9110\n",
      "recall:\t\t0.9390\n",
      "F1 score:\t0.9248\n",
      "\n",
      "=*= feature =*=\n",
      "precision:\t0.8950\n",
      "recall:\t\t0.8967\n",
      "F1 score:\t0.8959\n",
      "\n",
      "=*= question =*=\n",
      "precision:\t0.7313\n",
      "recall:\t\t0.5686\n",
      "F1 score:\t0.6397\n",
      "\n",
      "=*= documentation =*=\n",
      "precision:\t0.7589\n",
      "recall:\t\t0.6974\n",
      "F1 score:\t0.7268\n",
      "\n",
      "=*= micro averages =*=\n",
      "precision:\t0.8906\n",
      "recall:\t\t0.8906\n",
      "F1 score:\t0.8906\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset('csv', data_files={\"test\": \"test.csv\"}, delimiter='\\t')\n",
    "\n",
    "y_true = dataset[\"test\"][\"labels\"]\n",
    "id2label = {0: \"bug\", 1: \"feature\", 2: \"question\", 3: \"documentation\"}\n",
    "y_true = [id2label[y] for y in y_true]\n",
    "\n",
    "y_pred = list(tqdm(classifier(KeyDataset(dataset[\"test\"], \"text\")), total=len(y_true)))\n",
    "y_pred = [p[\"label\"] for p in y_pred]\n",
    "\n",
    "for label in [\"bug\", \"feature\", \"question\", \"documentation\"]:\n",
    "  P_c = sklearn.metrics.precision_score(y_true, y_pred, average=None, labels=[label])[0]\n",
    "  R_c = sklearn.metrics.recall_score(y_true, y_pred, average=None, labels=[label])[0]\n",
    "  F1_c = sklearn.metrics.f1_score(y_true, y_pred, average=None, labels=[label])[0]\n",
    "  print(f\"=*= {label} =*=\")\n",
    "  print(f\"precision:\\t{P_c:.4f}\")\n",
    "  print(f\"recall:\\t\\t{R_c:.4f}\")\n",
    "  print(f\"F1 score:\\t{F1_c:.4f}\")\n",
    "  print()\n",
    "\n",
    "\n",
    "P = sklearn.metrics.precision_score(y_true, y_pred, average='micro')\n",
    "R = sklearn.metrics.recall_score(y_true, y_pred, average='micro')\n",
    "F1 = sklearn.metrics.f1_score(y_true, y_pred, average='micro')\n",
    "\n",
    "print(\"=*= micro averages =*=\")\n",
    "print(f\"precision:\\t{P:.4f}\")\n",
    "print(f\"recall:\\t\\t{R:.4f}\")\n",
    "print(f\"F1 score:\\t{F1:.4f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "transformers"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}