2-cah_unsup_pretrain.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "94ae1e0b",
   "metadata": {},
   "source": [
    "## pretrain unsupervised sentence transformer model\n",
    "* on all text\n",
    "* unsup methods only\n",
    "\n",
    "\n",
    "`torch.cuda.empty_cache()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "experienced-isolation",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "pd.set_option(\"display.precision\", 4)\n",
    "pd.set_option('display.width', 1024)\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "%matplotlib inline \n",
    "\n",
    "from sklearn.model_selection import train_test_split,GridSearchCV, cross_val_score, cross_val_predict\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "\n",
    "from sklearn.metrics import classification_report, roc_auc_score  \n",
    "# https://stackoverflow.com/questions/53784971/how-to-disable-convergencewarning-using-sklearn\n",
    "from warnings import simplefilter\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "simplefilter(\"ignore\", category=ConvergenceWarning)\n",
    "\n",
    "from sentence_transformers import SentenceTransformer, util ## versions error? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "569cfb20",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, SentencesDataset, InputExample, losses, evaluation\n",
    "import torch\n",
    "torch.cuda.is_available()\n",
    "from torch.utils.data import DataLoader\n",
    "import torch.utils.data as data_utils\n",
    "## https://stackoverflow.com/questions/50307707/convert-pandas-dataframe-to-pytorch-tensor\n",
    "from transformers import AutoTokenizer, AutoModel\n",
    "## example from : https://www.sbert.net/docs/training/overview.html\n",
    "from sentence_transformers import SentenceTransformer, SentencesDataset, InputExample, losses \n",
    "\n",
    "from sentence_transformers import SentenceTransformer, LoggingHandler\n",
    "from sentence_transformers import models, util, datasets, evaluation, losses, SentencesDataset ## https://www.sbert.net/docs/package_reference/losses.html + SentencesDataset\n",
    "from sentence_transformers import SentenceTransformer, InputExample, losses\n",
    "from torch.utils.data import DataLoader\n",
    "from sentence_transformers import SentenceTransformer, SentencesDataset, losses ## MultipleNegativesRankingLoss\n",
    "## https://www.sbert.net/docs/package_reference/losses.html#multiplenegativesrankingloss\n",
    "# from sentence_transformers.readers import InputExample\n",
    "import logging\n",
    "import os\n",
    "import gzip\n",
    "from torch.utils.data import DataLoader\n",
    "from datetime import datetime\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "sound-transcript",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2.39 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>380151</th>\n",
       "      <td>Academy Award winner Meryl Streep. Betcha can't have just one!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433877</th>\n",
       "      <td>Next on ESPN2: The World Series of A man in yoga pants with a ponytail and feather earrings.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>769229</th>\n",
       "      <td>Unfortunately, no one can be told what A homoerotic volleyball montage is. You have to experience it for yourself.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5680</th>\n",
       "      <td>Don't worry, kid. It gets better. I've been living with A sales team of clowns and pedophiles for 20 years.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>812414</th>\n",
       "      <td>Hulu's new reality show features twelve hot singles living with Sucking the caviar straight out of a fish's pussy.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263092</th>\n",
       "      <td>Oh, whoops! These are not the right pants for A bass drop so huge it tears the starry vault asunder to reveal the face of God.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>832460</th>\n",
       "      <td>You Won't Believe These 15 Hilarious Becoming the President of the United States Bloopers!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>838690</th>\n",
       "      <td>There ain't no problem a little hard liquor and One titty hanging out can't fix.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42418</th>\n",
       "      <td>Armani suit: $1,000. Dinner for two at that swanky restaurant: $300. The look on her face when you surprise her with How good this cantaloupe feels on my penis: priceless.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767472</th>\n",
       "      <td>I don't believe in God. I believe in Assassinating the president.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>857056 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                  0\n",
       "380151                                                                                                               Academy Award winner Meryl Streep. Betcha can't have just one!\n",
       "433877                                                                                 Next on ESPN2: The World Series of A man in yoga pants with a ponytail and feather earrings.\n",
       "769229                                                           Unfortunately, no one can be told what A homoerotic volleyball montage is. You have to experience it for yourself.\n",
       "5680                                                                    Don't worry, kid. It gets better. I've been living with A sales team of clowns and pedophiles for 20 years.\n",
       "812414                                                           Hulu's new reality show features twelve hot singles living with Sucking the caviar straight out of a fish's pussy.\n",
       "...                                                                                                                                                                             ...\n",
       "263092                                               Oh, whoops! These are not the right pants for A bass drop so huge it tears the starry vault asunder to reveal the face of God.\n",
       "832460                                                                                   You Won't Believe These 15 Hilarious Becoming the President of the United States Bloopers!\n",
       "838690                                                                                             There ain't no problem a little hard liquor and One titty hanging out can't fix.\n",
       "42418   Armani suit: $1,000. Dinner for two at that swanky restaurant: $300. The look on her face when you surprise her with How good this cantaloupe feels on my penis: priceless.\n",
       "767472                                                                                                            I don't believe in God. I believe in Assassinating the president.\n",
       "\n",
       "[857056 rows x 1 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df = pd.read_csv(\"df_text.csv.gz\").sample(frac=1).drop_duplicates()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cb21c26e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    857056.0000\n",
       "mean         15.8096\n",
       "std           6.1033\n",
       "min           1.0000\n",
       "25%          11.0000\n",
       "50%          15.0000\n",
       "75%          19.0000\n",
       "max          57.0000\n",
       "Name: 0, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"0\"].str.split().str.len().describe() # min 1, max 57"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "81095e94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "850951\n"
     ]
    }
   ],
   "source": [
    "## drop mostly answers that have 1-2 words\n",
    "df = df.loc[(df[\"0\"].str.split().str.len()>4) & (df[\"0\"].str.len()>14)].drop_duplicates()\n",
    "print(df.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "compound-amino",
   "metadata": {},
   "source": [
    "## Sentence transformers model(s)\n",
    "* Sentence bert (NLI? semtantic similarity?) \n",
    "https://www.sbert.net/docs/training/overview.html\n",
    "https://www.sbert.net/docs/quickstart.html  , https://github.com/UKPLab/sentence-transformers\n",
    "\n",
    "\n",
    "\n",
    "List of pretrained sentencebert models:  \n",
    "* https://www.sbert.net/docs/pretrained_models.html\n",
    "* all-MiniLM-L12-v2\n",
    "* all-MiniLM-L6-v2\n",
    "* `stsb-distilroberta-base-v2` - 305M sized model - semantic similarity\n",
    "* `nli-distilroberta-base-v2` - NLI \n",
    "* `average_word_embeddings_glove.6B.300d`, `average_word_embeddings_glove.840B.300d` - glove/w2v embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7c4dbb54",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache() ## clear memory on rerun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "45535b42",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"all-MiniLM-L12-v2\"\n",
    "# model_name ='output/cah_tsdae-model'\n",
    "\n",
    "## \"all-MiniLM-L12-v2\"\n",
    "## \"all-MiniLM-L6-v2\"\n",
    "## paraphrase-MiniLM-L6-v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "consecutive-journalist",
   "metadata": {},
   "outputs": [],
   "source": [
    "## pretrained weights - download vis program often fails, easier to download externally from ftp\n",
    "# model = SentenceTransformer('stsb-distilroberta-base-v2') ## was distilbert-base-nli-mean-tokens\n",
    "# model = SentenceTransformer(\"nli-distilroberta-base-v2\",device=\"cuda\")\n",
    "# model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\") ## \"paraphrase-MiniLM-L12-v2\"\n",
    "\n",
    "model = SentenceTransformer(model_name,device=\"cuda\")\n",
    "\n",
    "# model = SentenceTransformer(\"./stsb-distilroberta-base-v2\",device=\"cuda\")\n",
    "\n",
    "# model = SentenceTransformer(\"./nli-mpnet-base-v2\",device=\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6026b18e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.max_seq_length # 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "777d4135",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[  101,  1996,  6346,  1010,  6203,  7987, 23998,  1010,  2003,  2025,\n",
       "           1999,  2256,  3340,  1010,  2021,  1999, 25120,  3348,  1012,   102]]),\n",
       " 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]),\n",
       " 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.tokenize([\"The fault, dear Brutus, is not in our stars, but in Meaningless sex.\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "534e4923",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.tokenize([\"The fault, dear Brutus, is not in our stars, but in Meaningless sex.\"])[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "legal-exemption",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "384"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_sentence_embedding_dimension() # 384"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3094c0db",
   "metadata": {},
   "source": [
    "## TSDAE pretraining\n",
    "https://www.sbert.net/examples/unsupervised_learning/TSDAE/README.html\n",
    "\n",
    "https://github.com/UKPLab/sentence-transformers/blob/master/examples/unsupervised_learning/TSDAE/train_tsdae_from_file.py\n",
    "* similar idea to MLM\n",
    "* Doesn't use pairs data or the like\n",
    "\n",
    "\n",
    "* L6, batch_size=64, amp : 33 minutes per tsdae epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b286fae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sentence_transformers import SentenceTransformer\n",
    "\n",
    "# model = SentenceTransformer(\"sentence-transformers/paraphrase-MiniLM-L12-v2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c765d703",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Define your sentence transformer model using CLS/mean (?) pooling\n",
    "# model_name = \"sentence-transformers/all-MiniLM-L12-v2\"\n",
    "# word_embedding_model = models.Transformer(model_name)\n",
    "# pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(), 'mean')\n",
    "# model = SentenceTransformer(modules=[word_embedding_model, pooling_model])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7942227d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Define a list with sentences (1k - 100k sentences)\n",
    "# train_sentences = [\"Your set of sentences\",\n",
    "#                    \"Model will automatically add the noise\", \n",
    "#                    \"And re-construct it\",\n",
    "#                    \"You should provide at least 1k sentences\"]\n",
    "\n",
    "# Create the special denoising dataset that adds noise on-the-fly\n",
    "train_dataset = datasets.DenoisingAutoEncoderDataset(df[\"0\"].values)\n",
    "\n",
    "# DataLoader to batch your data\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=40, shuffle=True)\n",
    "\n",
    "# Use the denoising auto-encoder loss\n",
    "train_loss = losses.DenoisingAutoEncoderLoss(model, decoder_name_or_path=model_name, tie_encoder_decoder=True)\n",
    "\n",
    "# Call the fit method\n",
    "model.fit(\n",
    "    train_objectives=[(train_dataloader, train_loss)],\n",
    "    epochs=7, ## ~1 hour per epoch\n",
    "    warmup_steps=2000,\n",
    "    weight_decay=0,\n",
    "    scheduler='constantlr',\n",
    "    optimizer_params={'lr': 6e-5},\n",
    "    show_progress_bar=True,\n",
    "    use_amp=True,\n",
    "    checkpoint_save_total_limit=5,\n",
    "    output_path=\"output\",\n",
    ")\n",
    "\n",
    "model.save('output/cah_tsdae-model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ab73ef81",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Define your sentence transformer model using CLS pooling\n",
    "# model_name = 'bert-base-uncased'\n",
    "# word_embedding_model = models.Transformer(model_name)\n",
    "# pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(), 'cls')\n",
    "# model = SentenceTransformer(modules=[word_embedding_model, pooling_model])\n",
    "\n",
    "# # Define a list with sentences (1k - 100k sentences)\n",
    "# train_sentences = [\"Your set of sentences\",\n",
    "#                    \"Model will automatically add the noise\", \n",
    "#                    \"And re-construct it\",\n",
    "#                    \"You should provide at least 1k sentences\"]\n",
    "\n",
    "# # Create the special denoising dataset that adds noise on-the-fly\n",
    "# train_dataset = datasets.DenoisingAutoEncoderDataset(train_sentences)\n",
    "\n",
    "# # DataLoader to batch your data\n",
    "# train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "\n",
    "# # Use the denoising auto-encoder loss\n",
    "# train_loss = losses.DenoisingAutoEncoderLoss(model, decoder_name_or_path=model_name, tie_encoder_decoder=True)\n",
    "\n",
    "# # Call the fit method\n",
    "# model.fit(\n",
    "#     train_objectives=[(train_dataloader, train_loss)],\n",
    "#     epochs=1,\n",
    "#     weight_decay=0,\n",
    "#     scheduler='constantlr',\n",
    "#     optimizer_params={'lr': 3e-5},\n",
    "#     show_progress_bar=True\n",
    "# )\n",
    "\n",
    "# model.save('output/tsdae-model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44eb33fa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}