CAH_finetune_DeepLearning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "liIk8f980xT5"
   },
   "source": [
    "# Sentence Embeddings using Siamese BERT-Networks/Sentence Transformers\n",
    "* Also TSDAE. pretraining? \n",
    "---\n",
    "This Google Colab Notebook illustrates using the Sentence Transformer python library to quickly create BERT embeddings for sentences and perform fast semantic searches.\n",
    "\n",
    "The Sentence Transformer library is available on [pypi](https://pypi.org/project/sentence-transformers/) and [github](https://github.com/UKPLab/sentence-transformers). The library implements code from the ACL 2019 paper entitled \"[Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410.pdf)\" by Nils Reimers and Iryna Gurevych.\n",
    "\n",
    "* https://github.com/UKPLab/sentence-transformers/blob/master/examples/unsupervised_learning/TSDAE/train_askubuntu_tsdae.py\n",
    "* https://www.sbert.net/examples/unsupervised_learning/TSDAE/README.html#tsdae-as-pre-training-task\n",
    "* https://www.sbert.net/docs/quickstart.html\n",
    "\n",
    "* https://www.sbert.net/docs/usage/semantic_textual_similarity.html\n",
    "\n",
    "\n",
    "TODO: finetune model:\n",
    "* https://stackoverflow.com/questions/63959319/bert-sentence-transformers-stops-quits-during-fine-tuning\n",
    "\n",
    "\n",
    "This version also combined white/black pairs and trains over a single column/text\n",
    "* https://datascience.stackexchange.com/questions/39345/how-to-replace-a-part-string-value-of-a-column-using-another-column\n",
    "* NOTE: Some cards lack a \"____\" - need to handle them "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FIjj9BjkY_A9",
    "outputId": "77d91f42-194e-4049-9a4c-7dd293569829"
   },
   "outputs": [],
   "source": [
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bzljhyTQEZds"
   },
   "source": [
    "## Install Sentence Transformer Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AmxRYxNDvn6y",
    "outputId": "023199f5-52cf-4a9e-80cb-cd3b60e58969"
   },
   "outputs": [],
   "source": [
    "# # Install the library using pip\n",
    "# !pip3 install sentence-transformers scikit-learn tensorflow -U\n",
    "# # !pip3 install sentence-transformers scikit-learn tensorflow tensorflow-text tf-models-official -U\n",
    "# # !pip3 install scikit-learn  -U\n",
    "# # !pip3 install nltk scikit-learn  -U\n",
    "\n",
    "# # import nltk\n",
    "# # nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "Av4muN8xC1C_"
   },
   "outputs": [],
   "source": [
    "#### TPU\n",
    "# VERSION = \"nightly\"  #@param [\"1.5\" , \"20200325\", \"nightly\"]\n",
    "# !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
    "# !python pytorch-xla-env-setup.py --version $VERSION\n",
    "\n",
    "# import torch\n",
    "# import torch_xla\n",
    "# import torch_xla.core.xla_model as xm\n",
    "\n",
    "# ### t = torch.randn(2, 2, device=xm.xla_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "jS9O0eAwlt8U"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n",
    "from sklearn.model_selection import train_test_split,GridSearchCV, cross_val_score, cross_val_predict\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.metrics import classification_report, roc_auc_score  \n",
    "\n",
    "# https://stackoverflow.com/questions/53784971/how-to-disable-convergencewarning-using-sklearn\n",
    "from warnings import simplefilter\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "simplefilter(\"ignore\", category=ConvergenceWarning)\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "FcJ95uWKXoJC"
   },
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, LoggingHandler\n",
    "from sentence_transformers import models, util, datasets, evaluation, losses, SentencesDataset ## https://www.sbert.net/docs/package_reference/losses.html + SentencesDataset\n",
    "from sentence_transformers import SentenceTransformer, InputExample, losses\n",
    "from torch.utils.data import DataLoader\n",
    "from sentence_transformers import SentenceTransformer, SentencesDataset, losses ## MultipleNegativesRankingLoss\n",
    "## https://www.sbert.net/docs/package_reference/losses.html#multiplenegativesrankingloss\n",
    "# from sentence_transformers.readers import InputExample\n",
    "import logging\n",
    "import os\n",
    "import gzip\n",
    "from torch.utils.data import DataLoader\n",
    "from datetime import datetime\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "1gRQUdycC9aR"
   },
   "outputs": [],
   "source": [
    "model_name = \"all-MiniLM-L12-v2\" #\"all-MiniLM-L6-v2\" # \"all-MiniLM-L12-v2\"\n",
    "# model_name = \"/content/drive/MyDrive/research/cah/cah_tsdae-model\" # prev\n",
    "min_cooccurences = 1 #4 # filter sentences for pairs that occurred at least k times. min 5: 200K. min 1: 1.9M\n",
    "\n",
    "ONE_COL_DATA_FORMAT = True#False\n",
    "USE_TEXT_COLS =  [\"text\"]#[\"text\",\t\"white_card_text\"]#[\"black_card_text\",\t\"white_card_text\"]\n",
    "\n",
    "FAST_RUN = False#True#False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eIAKz6KVEndZ"
   },
   "source": [
    "## Load the sBERT Model\n",
    "\n",
    "* Default , later try pretrained+ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "5IO_j2Ofv5pq"
   },
   "outputs": [],
   "source": [
    "# Load the BERT model. Various models trained on Natural Language Inference (NLI) https://github.com/UKPLab/sentence-transformers/blob/master/docs/pretrained-models/nli-models.md and \n",
    "# Semantic Textual Similarity are available https://github.com/UKPLab/sentence-transformers/blob/master/docs/pretrained-models/sts-models.md\n",
    "\n",
    "# model = SentenceTransformer('bert-base-nli-mean-tokens')\n",
    "# model = SentenceTransformer(\"nli-distilroberta-base-v2\")\n",
    "## # https://www.sbert.net/docs/pretrained_models.html\n",
    "# model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\") ## \"paraphrase-MiniLM-L12-v2\"\n",
    "# model = SentenceTransformer(\"paraphrase-MiniLM-L12-v2\")\n",
    "model = SentenceTransformer(model_name)\n",
    "# model = SentenceTransformer(\"all-MiniLM-L12-v2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "1b9dorxmsWtJ"
   },
   "outputs": [],
   "source": [
    "## copied from another notebook; may not be most up to date??\n",
    "\n",
    "def eval_preds(df_test,preds):\n",
    "    df2 = df_test[[\"fake_round_id\",\"won\"]].copy()\n",
    "    df2[\"preds\"] = preds\n",
    "    df2 = df2.sample(frac=1)\n",
    "    df2[\"m_score\"] = df2.groupby(\"fake_round_id\")[\"preds\"].transform(\"max\")\n",
    "    df2[\"correct\"] = ((df2[\"preds\"]==df2[\"m_score\"]) &(df2[\"won\"]>0)).astype(int)#df2.loc[df2[\"preds\"]==df2[\"m_score\"]\n",
    "    # df2.tail(31) ## there are cases wtih multiple values with same score/rank?  - ignore for now\n",
    "    df2.sort_values(\"preds\",ascending=False,inplace=True)\n",
    "    print(classification_report(df2[\"won\"],df2[\"correct\"]))\n",
    "    print(\"rocAUC:\",round(roc_auc_score(df2[\"won\"],df2[\"preds\"]),4))\n",
    "    print(f'top 1 acc by round: {100*df2.groupby(\"fake_round_id\").head(1)[\"won\"].mean():.2f}') # older ?\n",
    "    print(f'top 2 acc by round: {100*df2.groupby(\"fake_round_id\").head(2)[\"won\"].mean():.2f}')# older ?\n",
    "    print(f'top 3 acc by round: {100*df2.groupby(\"fake_round_id\").head(3)[\"won\"].mean():.2f}')# older ?\n",
    "    print(\"top 1 acc by round:\",round(df2.groupby(\"fake_round_id\").head(1)[\"won\"].mean(),4))\n",
    "    print(\"top 2 acc by round:\",round(df2.groupby(\"fake_round_id\").head(2).groupby(\"fake_round_id\")[\"won\"].max().mean(),4))\n",
    "    print(\"top 3 acc by round:\",round(df2.groupby(\"fake_round_id\").head(3).groupby(\"fake_round_id\")[\"won\"].max().mean(),4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I7_Ib3ITEwgO"
   },
   "source": [
    "## Setup a Corpus\n",
    "\n",
    "* `/content/drive/MyDrive/Research/CAH/cah_train_min7_v1.csv` - 37K\n",
    "    * `/content/drive/MyDrive/Research/CAH/cah_min7_v2.csv.gz`\n",
    "* `/content/drive/MyDrive/Research/CAH/cah_train_min6_v1.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "gT9VrZp1_mvO"
   },
   "outputs": [],
   "source": [
    "# KEEP_COLS = [\"black_card_text\",\"white_card_text\",\"text\",\"picks\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "K7tGxkF4l-oB",
    "outputId": "4c2637b9-8d86-4e51-e7c1-44f937749403"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1957080 rows\n",
      "             won  pair_count\n",
      "count  1957080.0  1957080.00\n",
      "mean         0.1        3.53\n",
      "std          0.3        1.60\n",
      "min          0.0        1.00\n",
      "25%          0.0        2.00\n",
      "50%          0.0        3.00\n",
      "75%          0.0        4.00\n",
      "max          1.0       13.00\n",
      "1957080\n"
     ]
    }
   ],
   "source": [
    "## \"/content/drive/MyDrive/Research/CAH/cah_min6_v2.csv.gz\"\n",
    "# \"/content/drive/MyDrive/Research/CAH/cah_min2_v2.csv.gz\"\n",
    "# \"/content/drive/MyDrive/Research/CAH/cah_train_min4_v1.csv\"\n",
    "# df = pd.read_csv(\"/content/drive/MyDrive/Research/CAH/cah_min6_v2.csv.gz\",usecols=[\"black_card_text\",\"white_card_text\",\"picks\"])#.sample(99)\n",
    "# df = pd.read_csv(\"/content/drive/MyDrive/Research/CAH/cah_train_min4_v1.csv\",usecols=[\"black_card_text\",\"white_card_text\",\"picks\"])#.sample(99)\n",
    "\n",
    "# df = pd.read_parquet(\"/content/drive/MyDrive/research/cah/cah_train_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\") \n",
    "df = pd.read_parquet(\"cah_train_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\") \n",
    "\n",
    "if FAST_RUN:\n",
    "    df = df.sort_values(\"fake_round_id\").head(11230)\n",
    "df[\"won\"] = df[\"won\"].astype(int)\n",
    "df = df.sort_values(by=\"won\",ascending=False) ## get picked pairs first\n",
    "\n",
    "# df_all = df.copy() ## copy for quick eval\n",
    "print(df.shape[0],\"rows\")\n",
    "\n",
    "df[\"pair_count\"] = df.groupby(\"text\")[\"won\"].transform(\"count\") ## can be used to filter sentences occurring less than k times\n",
    "print(df[[\"won\",\"pair_count\"]].describe().round(2) )\n",
    "\n",
    "# df = df.drop_duplicates(subset=[\"black_card_text\",\"white_card_text\"],keep=\"first\")#.sample(frac=1)\n",
    "print(df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# df_embeds = pd.read_parquet(\"cah_embed_L12.parquet\") ## existing file I have locally. is it good? \n",
    "# df_embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_embeds.isna().max(axis=0).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vyWPiyqQBphJ"
   },
   "source": [
    "Test set\n",
    "* Keeps round level grouping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 728
    },
    "id": "IxyngBX8ACCc",
    "outputId": "554ba40b-2191-4469-9459-3a4e40c21bf6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fake_round_id                48928\n",
      "black_card_text                580\n",
      "white_card_text               2118\n",
      "ID_index                        10\n",
      "id_white                      2118\n",
      "round_completion_seconds       705\n",
      "won                              2\n",
      "text                        377397\n",
      "sum_won                          9\n",
      "dtype: int64\n",
      "489280 rows\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fake_round_id</th>\n",
       "      <th>black_card_text</th>\n",
       "      <th>white_card_text</th>\n",
       "      <th>ID_index</th>\n",
       "      <th>id_white</th>\n",
       "      <th>round_completion_seconds</th>\n",
       "      <th>won</th>\n",
       "      <th>text</th>\n",
       "      <th>sum_won</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2387657</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>The NRA</td>\n",
       "      <td>8</td>\n",
       "      <td>1755</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387651</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Fisting</td>\n",
       "      <td>2</td>\n",
       "      <td>790</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387653</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Crab</td>\n",
       "      <td>4</td>\n",
       "      <td>590</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387654</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Slowly releasing a huge fart over the course o...</td>\n",
       "      <td>5</td>\n",
       "      <td>1591</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387655</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Elon Musk</td>\n",
       "      <td>6</td>\n",
       "      <td>721</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989543</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Holding up the line at Walgreens by trying to ...</td>\n",
       "      <td>4</td>\n",
       "      <td>997</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! Holding up the line at Walgreens by...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989545</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Breastfeeding a ten-year-old</td>\n",
       "      <td>6</td>\n",
       "      <td>495</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! Breastfeeding a ten-year-old killed...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989542</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>A bird that shits human turds</td>\n",
       "      <td>3</td>\n",
       "      <td>56</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! A bird that shits human turds kille...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989547</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Jerking off to a 10-second RealMedia clip</td>\n",
       "      <td>8</td>\n",
       "      <td>1077</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! Jerking off to a 10-second RealMedi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989541</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Jeff Bezos</td>\n",
       "      <td>2</td>\n",
       "      <td>1073</td>\n",
       "      <td>7613</td>\n",
       "      <td>1</td>\n",
       "      <td>Oh my god! Jeff Bezos killed Kenny!</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>489280 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         fake_round_id                                    black_card_text  \\\n",
       "2387657         238766  What's about to take this dance floor to the n...   \n",
       "2387651         238766  What's about to take this dance floor to the n...   \n",
       "2387653         238766  What's about to take this dance floor to the n...   \n",
       "2387654         238766  What's about to take this dance floor to the n...   \n",
       "2387655         238766  What's about to take this dance floor to the n...   \n",
       "...                ...                                                ...   \n",
       "2989543         298955                        Oh my god! __ killed Kenny!   \n",
       "2989545         298955                        Oh my god! __ killed Kenny!   \n",
       "2989542         298955                        Oh my god! __ killed Kenny!   \n",
       "2989547         298955                        Oh my god! __ killed Kenny!   \n",
       "2989541         298955                        Oh my god! __ killed Kenny!   \n",
       "\n",
       "                                           white_card_text  ID_index  \\\n",
       "2387657                                            The NRA         8   \n",
       "2387651                                            Fisting         2   \n",
       "2387653                                               Crab         4   \n",
       "2387654  Slowly releasing a huge fart over the course o...         5   \n",
       "2387655                                          Elon Musk         6   \n",
       "...                                                    ...       ...   \n",
       "2989543  Holding up the line at Walgreens by trying to ...         4   \n",
       "2989545                       Breastfeeding a ten-year-old         6   \n",
       "2989542                      A bird that shits human turds         3   \n",
       "2989547          Jerking off to a 10-second RealMedia clip         8   \n",
       "2989541                                         Jeff Bezos         2   \n",
       "\n",
       "         id_white  round_completion_seconds  won  \\\n",
       "2387657      1755                        46    0   \n",
       "2387651       790                        46    0   \n",
       "2387653       590                        46    1   \n",
       "2387654      1591                        46    0   \n",
       "2387655       721                        46    0   \n",
       "...           ...                       ...  ...   \n",
       "2989543       997                      7613    0   \n",
       "2989545       495                      7613    0   \n",
       "2989542        56                      7613    0   \n",
       "2989547      1077                      7613    0   \n",
       "2989541      1073                      7613    1   \n",
       "\n",
       "                                                      text  sum_won  \n",
       "2387657  What's about to take this dance floor to the n...        1  \n",
       "2387651  What's about to take this dance floor to the n...        0  \n",
       "2387653  What's about to take this dance floor to the n...        1  \n",
       "2387654  What's about to take this dance floor to the n...        1  \n",
       "2387655  What's about to take this dance floor to the n...        0  \n",
       "...                                                    ...      ...  \n",
       "2989543  Oh my god! Holding up the line at Walgreens by...        0  \n",
       "2989545  Oh my god! Breastfeeding a ten-year-old killed...        0  \n",
       "2989542  Oh my god! A bird that shits human turds kille...        0  \n",
       "2989547  Oh my god! Jerking off to a 10-second RealMedi...        0  \n",
       "2989541                Oh my god! Jeff Bezos killed Kenny!        3  \n",
       "\n",
       "[489280 rows x 9 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_test = pd.read_parquet(\"/content/drive/MyDrive/research/cah/cah_test_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\").sample(frac=1)\n",
    "df_test = pd.read_parquet(\"cah_test_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\")\n",
    "df_test[\"won\"] = df_test[\"won\"].astype(int)\n",
    "\n",
    "if FAST_RUN:\n",
    "    df_test = df_test.sort_values(\"fake_round_id\").head(230)\n",
    "print(df_test.nunique())\n",
    "print(df_test.shape[0],\"rows\")\n",
    "df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kUh_gOK32mZu"
   },
   "source": [
    "### Mean baseline priors\n",
    "* By min cooccurrences of sentence pairs in in raw data\n",
    "```\n",
    "  1 min occ Prior Acc: 0.2044\n",
    "  2 min occ Prior Acc: 0.2032\n",
    "  3 min occ Prior Acc: 0.2027\n",
    "  4 min occ Prior Acc: 0.2011\n",
    "  5 min occ Prior Acc: 0.1922\n",
    "  6 min occ Prior Acc: 0.1762\n",
    "  7 min occ Prior Acc: 0.1503\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "QRC7aCEE2mBe"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "  for i in range(1,8):\n",
    "    print(f\"{i} min occ, {df_temp.shape[0]} rows\")\n",
    "    df_temp = df.loc[df[\"pair_count\"] >=i].copy()\n",
    "    df_white_prior = df_temp.groupby([\"white_card_text\"], as_index=False)[\"won\"].mean().rename(columns={\"won\":\"white_prior\"}).set_index(\"white_card_text\")\n",
    "    df_test = df_test.join(df_white_prior,on=\"white_card_text\",how=\"left\")\n",
    "    prior = df_test[\"white_prior\"].mean()\n",
    "    df_test[\"white_prior\"] = df_test[\"white_prior\"].fillna(prior)\n",
    "    print(\"White Prior Acc: %.3f\" %df_test.sort_values(\"white_prior\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    \n",
    "  ## prior for a black-white combination - mean (freq% won), or sum (times won?) , or threshholded max? (over 2 times?)\n",
    "    df_pair_prior = df_temp.groupby([\"white_card_text\",\"black_card_text\"], as_index=False)[\"won\"].sum().rename(columns={\"won\":\"pair_prior\"}).set_index([\"white_card_text\",\"black_card_text\"])\n",
    "    \n",
    "    df_test = df_test.join(df_pair_prior,on=[\"white_card_text\",\"black_card_text\"],how=\"left\")\n",
    "    prior = df_test[\"pair_prior\"].mean()\n",
    "    df_test[\"pair_prior\"] = df_test[\"pair_prior\"].fillna(prior)\n",
    "    print(\"Pair Prior (Only)  Acc: %.3f\" %df_test.sort_values(\"pair_prior\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    print(\"White then Pair Prior Acc: %.3f\" %df_test.sort_values([\"white_prior\",\"pair_prior\"],ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    print(\"Pair Prior then White Acc: %.3f\" %df_test.sort_values([\"pair_prior\",\"white_prior\",],ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    \n",
    "    df_test.drop(columns=[\"white_prior\",\"pair_prior\"],errors=\"ignore\",inplace=True)\n",
    "except:()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zhvJzhL2BeFM"
   },
   "source": [
    "Drop duplicate instances with same out put (i.e ignore round level/ranking) \n",
    "* keep positives preferrably\n",
    "* Could do : weight or filter bby # occurrences\n",
    "\n",
    "* 22% mean win rate after this (instea of 10%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "OKAwvWRzBctu"
   },
   "outputs": [],
   "source": [
    "if min_cooccurences>1:\n",
    "  ## filter for sentence pairs occurring at least X times, regardless of label\n",
    "  df = df.loc[df[\"pair_count\"] >=min_cooccurences]\n",
    "  print(df.nunique())\n",
    "  print(df.shape[0],f\"rows after {min_cooccurences} filter of pairs\")\n",
    "  df = df.drop(columns=[\"fake_round_id\",\"prior_white\",\"pair_count\"],errors=\"ignore\") # drop round id if not doing group level ranking\n",
    "  try:\n",
    "    df = df.sort_values(by=\"won\",ascending=False).drop_duplicates(subset=[\"black_card_text\",\"white_card_text\"],keep=\"first\")\n",
    "    df_test = df_test.sort_values(by=\"won\",ascending=False).drop_duplicates(subset=[\"black_card_text\",\"white_card_text\"],keep=\"first\")\n",
    "  except: print(\"sort by and keep first failed\")\n",
    "\n",
    "  print(\"mean won:\",df[\"won\"].mean())\n",
    "  print(df.nunique())\n",
    "  print(df.shape[0],\"rows\")\n",
    "  df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "_1Wx14J-6uV1"
   },
   "outputs": [],
   "source": [
    "df = df.sample(frac=1)\n",
    "df_test = df_test.sample(frac=1)\n",
    "\n",
    "df.reset_index(inplace=True,drop=True)\n",
    "df_test.reset_index(inplace=True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "szXA3q0VWDjg"
   },
   "outputs": [],
   "source": [
    "# import csv\n",
    "# df[[#'black_card_text',\n",
    "#     # 'white_card_text',\n",
    "#   'text','won']].rename(columns={'won':\"target\"}).to_csv(f\"cah_min{min_cooccurences}.csv.gz\",compression=\"gzip\",index=False,quoting=csv.QUOTE_ALL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JUVdkcJ7hFu3"
   },
   "source": [
    "#### train - eval split (if doing supervised pretraining... )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PLDoriAMFDRE"
   },
   "source": [
    "##### Data formats: \n",
    "1. 2 text cols\n",
    "  * Can be white, black\n",
    "  * could be merged (`text`) and white/black,\n",
    "2. single joint text col\n",
    "  * Merged (`text`) col\n",
    "\n",
    "\n",
    "\n",
    "  https://www.pinecone.io/learn/train-sentence-transformers-softmax/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "9k8qHhym2x2w"
   },
   "outputs": [],
   "source": [
    "y = df[\"won\"].values\n",
    "\n",
    "if ONE_COL_DATA_FORMAT:\n",
    "  ### 1 text col version\n",
    "  ## orig:\n",
    "  # X_train, X_test, y_train, y_test = train_test_split(\n",
    "  #   list(df[\"text\"].values), list(df[\"won\"].astype(int).values), test_size=0.2, random_state=42)\n",
    "## NEW/ALT - use defined train, test:\n",
    "  X_train=list(df[\"text\"].values)\n",
    "  y_train = list(df[\"won\"].values)\n",
    "  X_test=list(df_test[\"text\"].values)\n",
    "  y_test = list(df_test[\"won\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "Msv2dCOphFIv"
   },
   "outputs": [],
   "source": [
    "# # X = np.concatenate([s1_emb,s2_emb],axis=1) # BOTH \n",
    "# # X = list(df[\"text\"].values)\n",
    "\n",
    "# if ONE_COL_DATA_FORMAT:\n",
    "#   ### 1 text col version\n",
    "#   ## orig:\n",
    "#   # X_train, X_test, y_train, y_test = train_test_split(\n",
    "#   #   list(df[\"text\"].values), list(df[\"won\"].astype(int).values), test_size=0.2, random_state=42)\n",
    "# ## NEW/ALT - use defined train, test:\n",
    "#   X_train=list(df[\"text\"].values)\n",
    "#   y_train = list(df[\"won\"].values)\n",
    "#   X_test=list(df_test[\"text\"].values)\n",
    "#   y_test = list(df_test[\"won\"].values)\n",
    "  \n",
    "#   train_samples=  [InputExample(texts=[X_train[i]],label=float(y_train[i])) for i in range(len(X_train))]\n",
    "#   test_samples=  [InputExample(texts=[X_test[i]],label=float(y_test[i])) for i in range(len(X_test))]\n",
    "\n",
    "#   ## added: copied from 2 text - expects 2 cols??\n",
    "#   ## LabelAccuracyEvaluator\n",
    "#   dev_eval = evaluation.BinaryClassificationEvaluator(sentences1=[x for x in X_test],sentences2 = [x for x in X_test],labels = y_test,\n",
    "#                                                                           batch_size=128,show_progress_bar=True,write_csv=True)\n",
    "# else:\n",
    "#   ### 2 text col version ; can try different cols\n",
    "#   \"\"\"\n",
    "#   ##ORIG:\n",
    "#   X_train, X_test, y_train, y_test = train_test_split(\n",
    "#     list(df[USE_TEXT_COLS].values), list(df[\"won\"].values), test_size=0.2, random_state=42)\n",
    "#   \"\"\"\n",
    "#   ## NEW/ALT - use defined train, test:\n",
    "#   X_train=list(df[USE_TEXT_COLS].values)\n",
    "#   y_train = list(df[\"won\"].values)\n",
    "#   X_test=list(df_test[USE_TEXT_COLS].values)\n",
    "#   y_test = list(df_test[\"won\"].values)\n",
    "#   train_samples=  [InputExample(texts=[X_train[i][0],X_train[i][1]],label=float(y_train[i])) for i in range(len(X_train))]\n",
    "#   test_samples=  [InputExample(texts=[X_test[i][0],X_test[i][1]],label=float(y_test[i])) for i in range(len(X_test))]\n",
    "#   ### binary evaluator: expect list 1, list 2:\n",
    "#   ## https://www.sbert.net/docs/package_reference/evaluation.html\n",
    "#   ## leaky: \n",
    "#   dev_eval = evaluation.BinaryClassificationEvaluator(sentences1=[x[0] for x in X_test],sentences2 = [x[1] for x in X_test],labels = y_test,\n",
    "#                                                                           batch_size=128,show_progress_bar=True,write_csv=True)\n",
    "# # train_examples =  [InputExample(texts=[X_train[i]],label=float(y_train[i])) for i in range(len(X_train))]\n",
    "# train_examples = X_train\n",
    "# test_examples = X_test # added"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ck8HAVmMZRli"
   },
   "source": [
    "* See how well unsupervised model does? \n",
    "* Check if funny combs are close or far"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "GO_PRyRAhVN6"
   },
   "outputs": [],
   "source": [
    "# model2 = SentenceTransformer(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "fZpw55M5ZaN2"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### could speed this up n^2 by getting embeddings only for unique combniations, then rejoining/merging...  (Won't work if using \"Text\" col)\n",
    "# s1_emb = model.encode(list(df[USE_TEXT_COLS[0]].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=128\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )\n",
    "\n",
    "# ## black cards = quesiton/prompt\n",
    "# s2_emb = model.encode(list(df[USE_TEXT_COLS[1]].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=256\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n",
    "\n",
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0]))\n",
    "# print(len(dot_sim))\n",
    "\n",
    "# df[\"dot_sim_score\"] = dot_sim\n",
    "# # df_all[\"dot_sim_score\"] = dot_sim\n",
    "# print(df[[\"dot_sim_score\",\"won\"]].corr())\n",
    "# print(\"df:\\n\",df.groupby(\"won\")[\"dot_sim_score\"].mean())\n",
    "\n",
    "# # print(\"df_All:\")\n",
    "# # print(df_all[[\"dot_sim_score\",\"won\"]].corr())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "uMXZ6ctoh-2-"
   },
   "outputs": [],
   "source": [
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0]))\n",
    "# print(len(dot_sim))\n",
    "\n",
    "# df[\"dot_sim_score\"] = dot_sim\n",
    "# # df_all[\"dot_sim_score\"] = dot_sim\n",
    "# print(df[[\"dot_sim_score\",\"won\"]].corr())\n",
    "# print(\"df:\\n\",df.groupby(\"won\")[\"dot_sim_score\"].mean())\n",
    "# print(\"df_all:\\n\",df_all.groupby(\"won\")[\"dot_sim_score\"].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OHa6wGqNghi5"
   },
   "source": [
    "### try supervised model\n",
    "\n",
    "* Contrastive loss?\n",
    "* https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/quora_duplicate_questions/training_OnlineContrastiveLoss.py\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html#training\n",
    "\n",
    "\n",
    "Can combine multiple losses:\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "o7Gzd4DmkB0N"
   },
   "outputs": [],
   "source": [
    "#As distance metric, we use cosine distance (cosine_distance = 1-cosine_similarity)\n",
    "distance_metric = losses.SiameseDistanceMetric.COSINE_DISTANCE\n",
    "\n",
    "#Negative pairs should have a distance of at least 0.3 (was 0.5 orig)\n",
    "margin = 0.5\n",
    "\n",
    "# ####  Configure the training #### \n",
    "# warmup_steps = math.ceil(len(train_dataset) * num_epochs / batch_size * 0.1) # 10% of train data for warm-up\n",
    "# print(\"Warmup-steps: {}\".format(warmup_steps))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "kJWvGNY7BJhY"
   },
   "outputs": [],
   "source": [
    "# ## results before finetuning\n",
    "# try:dev_eval(model) ## % with 7e-4 and min 4\n",
    "# except:()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 142
    },
    "id": "OuDei5zLgWFR",
    "outputId": "abbf863f-b5ef-4b42-b5f7-87e620b90f8b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ntrain_dataset = SentencesDataset(train_samples, model=model)\\n# DataLoader to batch your data\\ntrain_dataloader = DataLoader(train_dataset, batch_size= 128,#128,\\n                              shuffle=True)\\n## OnlineContrastiveLoss. - expects pairs\\n# train_loss = losses.OnlineContrastiveLoss(model=model, distance_metric=distance_metric, margin=margin)\\ntrain_loss = losses.CosineSimilarityLoss(model=model) #ORIG\\n\\n##ALT:\\ntrain_loss = losses.SoftmaxLoss(model=model,num_labels=1, sentence_embedding_dimension=384)\\n# Call the fit method\\ntry:\\n  model.fit(\\n      train_objectives=[(train_dataloader, train_loss)],\\n      epochs=1,\\n      # weight_decay=0,\\n      # scheduler='constantlr', \\n      optimizer_params={'lr': 1e-4 ## 3e-5\\n                        },\\n      show_progress_bar=True,\\n      use_amp=True,\\n      # evaluator=dev_eval, # ORIG - \\n      output_path='./cah_sbert_cos',\\n  )\\nexcept:()\\n\""
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "train_dataset = SentencesDataset(train_samples, model=model)\n",
    "# DataLoader to batch your data\n",
    "train_dataloader = DataLoader(train_dataset, batch_size= 128,#128,\n",
    "                              shuffle=True)\n",
    "## OnlineContrastiveLoss. - expects pairs\n",
    "# train_loss = losses.OnlineContrastiveLoss(model=model, distance_metric=distance_metric, margin=margin)\n",
    "train_loss = losses.CosineSimilarityLoss(model=model) #ORIG\n",
    "\n",
    "##ALT:\n",
    "train_loss = losses.SoftmaxLoss(model=model,num_labels=1, sentence_embedding_dimension=384)\n",
    "# Call the fit method\n",
    "try:\n",
    "  model.fit(\n",
    "      train_objectives=[(train_dataloader, train_loss)],\n",
    "      epochs=1,\n",
    "      # weight_decay=0,\n",
    "      # scheduler='constantlr', \n",
    "      optimizer_params={'lr': 1e-4 ## 3e-5\n",
    "                        },\n",
    "      show_progress_bar=True,\n",
    "      use_amp=True,\n",
    "      # evaluator=dev_eval, # ORIG - \n",
    "      output_path='./cah_sbert_cos',\n",
    "  )\n",
    "except:()\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 55
    },
    "id": "ufLRWUIWkZ-I",
    "outputId": "d85b744e-05bd-404e-fcea-4956080a03e3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n### dev eval\\n## example: https://jovian.ai/vumichien/sbert\\n# dev_eval(model) ## 0.555 Accracy (With min 6) , 57% with 1e-3 and min 3\\ntry:dev_eval(model) ## % with 7e-4 and min 4\\nexcept:()\\n'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "### dev eval\n",
    "## example: https://jovian.ai/vumichien/sbert\n",
    "# dev_eval(model) ## 0.555 Accracy (With min 6) , 57% with 1e-3 and min 3\n",
    "try:dev_eval(model) ## % with 7e-4 and min 4\n",
    "except:()\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84,
     "referenced_widgets": [
      "cfea03270fe64a44b54cde507f629eaf",
      "3f969476da554298b79fb91c4ec6e08f",
      "24c0db6a9c2146bea1b658c7245044aa",
      "82841c43fda44080b6debfd95e425b0f",
      "82cc1edd77f047b084a5f0c0b17a4bb1",
      "9952f2e5f62d402eadd85c0c17d5d8c2",
      "12f0eead030b48e79d989ab53ae75f5a",
      "ef2b365a6ece4bdb8d9cb84b6e593047",
      "35d4592d40f344ae92e27bd0bdc0f129",
      "d54fadb082c64e1e868ed9b21bdb4698",
      "9933a11516d64e00a16d8793fbb2ac04"
     ]
    },
    "id": "_CDwk0GxdAk0",
    "outputId": "3079a3bb-4ad0-4a70-e22a-67add369d1a4"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8602194e90684ccfb8f8bdee63e9fe18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/7645 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 20min 20s\n",
      "Wall time: 15min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "### could speed this up n^2 by getting embeddings only for unique combniations, then rejoining/merging...  (Won't work if using \"Text\" col)\n",
    "### ~ 1.5 minutes on colab GPU for 123K train samples , L6.  ~ 30-42 min with L12, full data\n",
    "### on 2060 local GPU, ~ 17 min for all train\n",
    "s1_emb = model.encode(list(df[USE_TEXT_COLS[0]].values),\n",
    "                      normalize_embeddings=False,\n",
    "                      show_progress_bar=True,\n",
    "                      batch_size=256,\n",
    "                    #   convert_to_tensor=True\n",
    "                    )\n",
    "if not ONE_COL_DATA_FORMAT:\n",
    "  ## black cards = quesiton/prompt\n",
    "  s2_emb = model.encode(list(df[USE_TEXT_COLS[1]].values),\n",
    "                        normalize_embeddings=False,\n",
    "                        show_progress_bar=True,\n",
    "                        batch_size=200,\n",
    "                      #   convert_to_tensor=True\n",
    "                      )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "04SnpNCFeDb4"
   },
   "outputs": [],
   "source": [
    "# try:\n",
    "#   dot_sim = []\n",
    "#   cos_score = []\n",
    "#   for i in range (len(s1_emb)):\n",
    "#       # dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "#       dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0])) ## am I getting the right cell with these two?? (cos and dot, [i][j] ? )\n",
    "#       cos_score.append(float(util.cos_sim(s1_emb[i], s2_emb[i])[0]))\n",
    "#   print(len(dot_sim))\n",
    "\n",
    "#   df[\"dot_sim_score\"] = dot_sim\n",
    "#   df[\"cos_sim_score\"] = cos_score\n",
    "#   print(df[[\"dot_sim_score\",\"cos_sim_score\",\"won\"]].corr().round(3))\n",
    "\n",
    "#   print(\"df:\\n\",df.groupby(\"won\")[[\"dot_sim_score\",\"cos_sim_score\"]].mean().round(3))\n",
    "\n",
    "#   # print(\"df_All:\")\n",
    "#   # df_all[\"dot_sim_score\"] = dot_sim\n",
    "#   # print(df_all[[\"dot_sim_score\",\"won\"]].corr())\n",
    "# except:()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4eT_LQFIiOhb"
   },
   "source": [
    "## Eval on df_Test\n",
    "* Can speed up by getting unique sentence pairs and their embeddings\n",
    "* Won't work the same for single text "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "Bv4dAPxkiXQK"
   },
   "outputs": [],
   "source": [
    "# df_test.drop_duplicates([\"black_card_text\",\"white_card_text\"]).shape[0] # 367K rows, vs 489 K for all "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84,
     "referenced_widgets": [
      "0b2804c196ef4f369aa469a3080cb930",
      "b17a4f8b6fca414286b47a65e60d1254",
      "6b8fcd36f1bb4a22981f0162ae538d7f",
      "62bb978e5a3b4a9cab47a8856c3633d0",
      "2941cf6f1bf745b7a8518ce2b1e8aaf7",
      "9f284c9e3c364f778c41b8c7e0ba3045",
      "030325446b8e4ede91020a4536f91064",
      "8c44dbe9d1c74aae894b1ff86ae69c9b",
      "75d0eb084e1b429b9375196c6069c55d",
      "b597b302d1d9405fbf9181d8a62e3944",
      "a5e9540e89ec4bee85fb45235c2fc640"
     ]
    },
    "id": "A6H_8-Ldirmw",
    "outputId": "6bada940-9caf-4a82-b16a-821d69afde77"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4356e3d57cd14504aa7463af670ad27d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1912 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 5min 10s\n",
      "Wall time: 3min 48s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "### could speed this up n^2 by getting embeddings only for unique combniations, then rejoining/merging...  (Won't work if using \"Text\" col)\n",
    "## change s1_emb to s1_emb_test\n",
    "## ~ 6 min for L12, local GPU, full test\n",
    "s1_emb_test = model.encode(list(df_test[USE_TEXT_COLS[0]].values),\n",
    "                      normalize_embeddings=False,\n",
    "                      show_progress_bar=True,\n",
    "                      batch_size=256,\n",
    "                    #   convert_to_tensor=True\n",
    "                    )\n",
    "if not ONE_COL_DATA_FORMAT:\n",
    "  ## black cards = quesiton/prompt\n",
    "  s2_emb_test = model.encode(list(df_test[USE_TEXT_COLS[1]].values),\n",
    "                        normalize_embeddings=False,\n",
    "                        show_progress_bar=True,\n",
    "                        batch_size=256,\n",
    "                      #   convert_to_tensor=True\n",
    "                      )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QWBwiv7mirq-",
    "outputId": "ec516967-cfca-41c1-8dc0-7ce8afc82ace"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if not ONE_COL_DATA_FORMAT:\n",
    "  try:\n",
    "    dot_sim = []\n",
    "    cos_score = []\n",
    "    for i in range (len(s1_emb)):\n",
    "        # dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "        dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0])) ## am I getting the right cell with these two?? (cos and dot, [i][j] ? )\n",
    "        cos_score.append(float(util.cos_sim(s1_emb[i], s2_emb[i])[0]))\n",
    "    print(len(dot_sim))\n",
    "\n",
    "    df_test[\"dot_sim_score\"] = dot_sim\n",
    "    df_test[\"cos_sim_score\"] = cos_score\n",
    "    print(df_test[[\"dot_sim_score\",\"cos_sim_score\",\"won\"]].corr().round(3))\n",
    "\n",
    "    print(\"df_test:\\n\",df_test.groupby(\"won\")[[\"dot_sim_score\",\"cos_sim_score\"]].mean().round(3))\n",
    "\n",
    "    print(\"Acc:\",df_test.sort_values(\"dot_sim_score\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.5% acc\n",
    "    print(\"Acc: (dot)\",df_test.sort_values(\"dot_sim_score\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean())\n",
    "    print(\"Acc:\",df_test.sort_values(\"cos_sim_score\",ascending=True).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 0.05\n",
    "    print(\"Acc:\",df_test.sort_values(\"cos_sim_score\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean())\n",
    "  except:()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rkIPD7_uXw6D"
   },
   "source": [
    "### Pretrain - unsupervised\n",
    "* TSDAE or other method ? \n",
    "* https://www.sbert.net/examples/unsupervised_learning/TSDAE/README.html#tsdae-as-pre-training-task\n",
    "\n",
    "\n",
    "ST recommends MultipleNegativesRankingLoss\n",
    "* https://www.sbert.net/examples/training/nli/README.html#multiplenegativesrankingloss\n",
    "*  MultipleNegativesRankingLoss only requires positive pairs, i.e., we only need examples of positive/funny pairs. (BUT It also supports hard negatives in a triplet)\n",
    "\n",
    "Also: OnlineContrastiveLoss\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html#training\n",
    "* Constrative Loss / `losses.OnlineContrastiveLoss`\n",
    "* Choosing the distance function and especially choosing a sensible margin are quite important for the success of constrative loss. In the given example, we use cosine_distance (which is 1-cosine_similarity) with a margin of 0.5. I.e., non-duplicate questions should have a cosine_distance of at least 0.5 (which is equivalent to a 0.5 cosine similarity difference).\n",
    "* An improved version of constrative loss is OnlineConstrativeLoss, which looks which negative pairs have a lower distance that the largest positive pair and which positive pairs have a higher distance than the lowest distance of negative pairs. I.e., this loss automatically detects the hard cases in a batch and computes the loss only for these cases.\n",
    "\n",
    "Can also do BOTH losses, as in:\n",
    "\n",
    "* Multi-Task-Learning\n",
    "    Constrative Loss works well for pair classification, i.e., given two pairs, are these duplicates or not. It pushes negative pairs far away in vector space, so that the distinguishing between duplicate and non-duplicate pairs works good.\n",
    "\n",
    "    MultipleNegativesRankingLoss on the other sides mainly reduces the distance between positive pairs out of large set of possible candidates. However, the distance between non-duplicate questions is not so large, so that this loss does not work that weill for pair classification.\n",
    "\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html#multi-task-learning\n",
    "\n",
    "More losses (e.g. triplet): https://www.sbert.net/docs/package_reference/losses.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "CAlPTP7_XwVp"
   },
   "outputs": [],
   "source": [
    "# # Define your sentence transformer model using CLS pooling\n",
    "# model_name = 'bert-base-uncased'\n",
    "# word_embedding_model = models.Transformer(model_name)\n",
    "# pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(), 'cls')\n",
    "# model = SentenceTransformer(modules=[word_embedding_model, pooling_model])\n",
    "\n",
    "# # Define a list with sentences (1k - 100k sentences)\n",
    "# train_sentences = [\"Your set of sentences\",\n",
    "#                    \"Model will automatically add the noise\", \n",
    "#                    \"And re-construct it\",\n",
    "#                    \"You should provide at least 1k sentences\"]\n",
    "\n",
    "# # Create the special denoising dataset that adds noise on-the-fly\n",
    "# train_dataset = datasets.DenoisingAutoEncoderDataset(train_sentences)\n",
    "\n",
    "# # DataLoader to batch your data\n",
    "# train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "\n",
    "# # Use the denoising auto-encoder loss\n",
    "# train_loss = losses.DenoisingAutoEncoderLoss(model, decoder_name_or_path=model_name, tie_encoder_decoder=True)\n",
    "\n",
    "# # Call the fit method\n",
    "# model.fit(\n",
    "#     train_objectives=[(train_dataloader, train_loss)],\n",
    "#     epochs=1,\n",
    "#     weight_decay=0,\n",
    "#     scheduler='constantlr',\n",
    "#     optimizer_params={'lr': 3e-5},\n",
    "#     show_progress_bar=True\n",
    "# )\n",
    "\n",
    "# model.save('output/tsdae-model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8UwMrw__mpbH"
   },
   "source": [
    "#### get embeddings & dot product/codine distance over all \n",
    "\n",
    "* `util.semantic_search` - could do this is one step for us (but we'll want to train a model on embeddings anyway) \n",
    "\n",
    "* https://www.sbert.net/docs/usage/semantic_textual_similarity.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "h-4YyJH1mwOJ"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ## white cards = most predictive information\n",
    "\n",
    "# # ### encode_multi_process ? (needs pool enabled)\n",
    "# ##     pool = model.start_multi_process_pool(encode_batch_size=2000)\n",
    "# ##     embeddings = model.encode_multi_process(texts, pool)\n",
    "\n",
    "# s1_emb = model.encode(list(df[\"white_card_text\"].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=128,\n",
    "#                     #   convert_to_tensor=True \n",
    "#                       )#.astype(np.float32) ## 40s for 4K , with L12 miniLM, on cpu\n",
    "# ### 7 min for all min7 (37k) with L12\n",
    "# print(s1_emb.shape,\"s1_emb.shape\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "c84GZCIOm6rc"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### 8.7 min on colab for L12, min 4 (371K)\n",
    "\n",
    "# s1_emb = model.encode(list(df[\"white_card_text\"].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=128,\n",
    "#                       normalize_embeddings=True\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )\n",
    "\n",
    "# ## black cards = quesiton/prompt\n",
    "# s2_emb = model.encode(list(df[\"black_card_text\"].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=256,\n",
    "#                       normalize_embeddings=True\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n",
    "# print(s2_emb.shape)\n",
    "\n",
    "\n",
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "# print(len(dot_sim))\n",
    "# dot_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "-HS3rT63uW6J"
   },
   "outputs": [],
   "source": [
    "# len(s2_emb[0]) # 384 = dim of the \"first\" sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "LUejcarOuocO"
   },
   "outputs": [],
   "source": [
    "# ### Hopefully I am doing this correctly ??? \n",
    "# ### get cosine similarity \n",
    "# output = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     output.append(util.cos_sim(s1_emb[i], s2_emb[i])[0])\n",
    "# print(len(output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZQYpl70Cn9aP",
    "outputId": "2426f254-606d-424b-ba33-b2441a3a81ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Compute cosine similarity between all pairs - outputs matrix of shape S1 X S2 (i.e # samples = ineffecient in memory!)\n",
    "# output = util.pytorch_cos_sim(s1_emb, s2_emb)\n",
    "# output = util.cos_sim(s1_emb, s2_emb) # ORIG, nXn matrix\n",
    "# print(output.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "g8plFSPvsN2C"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### Hopefully I am doing this correctly ??? \n",
    "# # # ## dot product:\n",
    "# # dot_sim = util.dot_score(s1_emb, s2_emb) # memory crash? \n",
    "# # print(dot_sim.shape)\n",
    "\n",
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "# print(len(dot_sim))\n",
    "# dot_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "4dO5Fro2AEJY"
   },
   "outputs": [],
   "source": [
    "# print(\"cos done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7ewqEcHfqgGW",
    "outputId": "618ae015-3984-4c2f-d169-8fe373bad667"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# from torch.nn import CosineSimilarity, PairwiseDistance\n",
    "# ## https://pytorch.org/docs/stable/generated/torch.nn.CosineSimilarity.html\n",
    "# nn_cos = CosineSimilarity(dim=1, eps=1e-6)\n",
    "# output = nn_cos(s1_emb, s2_emb) \n",
    "# print(\"output\",output.shape)\n",
    "# nn_pairwiseDist = PairwiseDistance()\n",
    "# output_2 = nn_pairwiseDist(s1_emb, s2_emb)\n",
    "# print(\"output_2\",output_2.shape)\n",
    "\n",
    "# # ## dot product:\n",
    "# output_3 = util.dot_score(s1_emb, s2_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "5bpVLWPqqgM4"
   },
   "outputs": [],
   "source": [
    "# df[\"cos_sim\"] = output\n",
    "# df[\"pairwiseDist_sim\"] = output_2\n",
    "# df[\"dot_score\"] = output_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "_pV02TX0q9H8"
   },
   "outputs": [],
   "source": [
    "# df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "4mJQsyAPrIPh"
   },
   "outputs": [],
   "source": [
    "# df.groupby(\"picks\")[[\"cos_sim\",\"pairwiseDist_sim\"]].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Uj7QLQmVt-7S"
   },
   "source": [
    "#### Model on embeddings \n",
    "* linear model on embeddings per sentence and cossim, pairwise dist score;\n",
    "* + difference, +- multiplication of vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "6kZfxLXWEF93"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### mean diff, math/ mult per row: \n",
    "\n",
    "# ## https://stackoverflow.com/questions/50430585/mean-difference-of-two-numpy-arrays\n",
    "# # np.mean(np.abs(s1_emb[:, None] - s2_emb))\n",
    "# vector_diffs = s1_emb - s2_emb\n",
    "# mean_diff = np.mean(vector_diffs,axis=1) # 1 col\n",
    "# max_diff = np.min(vector_diffs,axis=1) \n",
    "# min_diff = np.max(vector_diffs,axis=1) \n",
    "# # np_dot = np.dot(s1_emb,s2_emb)\n",
    "# print(mean_diff)\n",
    "# print(max_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 641
    },
    "id": "_T3ooMlWiVsT",
    "outputId": "d89f8a79-64c2-4aa1-9bc4-11a64a474150"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fake_round_id</th>\n",
       "      <th>black_card_text</th>\n",
       "      <th>white_card_text</th>\n",
       "      <th>ID_index</th>\n",
       "      <th>id_white</th>\n",
       "      <th>round_completion_seconds</th>\n",
       "      <th>won</th>\n",
       "      <th>text</th>\n",
       "      <th>sum_won</th>\n",
       "      <th>pair_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>162435</td>\n",
       "      <td>Can a woman really have it all? A career and __?</td>\n",
       "      <td>Bad bitches with fat asses</td>\n",
       "      <td>4</td>\n",
       "      <td>392</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>Can a woman really have it all? A career and B...</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>76153</td>\n",
       "      <td>Well if you'll excuse me, gentlemen, I have a ...</td>\n",
       "      <td>Seizing control of the means of production</td>\n",
       "      <td>4</td>\n",
       "      <td>1536</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>Well if you'll excuse me, gentlemen, I have a ...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>184382</td>\n",
       "      <td>Here at the Academy for Gifted Children, we al...</td>\n",
       "      <td>The Devil himself</td>\n",
       "      <td>9</td>\n",
       "      <td>1739</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>Here at the Academy for Gifted Children, we al...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>57103</td>\n",
       "      <td>Armani suit: $1,000. Dinner for two at that sw...</td>\n",
       "      <td>The vagina hole</td>\n",
       "      <td>5</td>\n",
       "      <td>1919</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Armani suit: $1,000. Dinner for two at that sw...</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>99201</td>\n",
       "      <td>Hey, honey. For the bedroom, were you thinking...</td>\n",
       "      <td>How awesome it is to be white</td>\n",
       "      <td>9</td>\n",
       "      <td>1015</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>Hey, honey. For the bedroom, were you thinking...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957075</th>\n",
       "      <td>218472</td>\n",
       "      <td>Come to Dubai, where you can relax in our worl...</td>\n",
       "      <td>New DNA evidence exonerating OJ Simpson</td>\n",
       "      <td>10</td>\n",
       "      <td>1315</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>Come to Dubai, where you can relax in our worl...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957076</th>\n",
       "      <td>197764</td>\n",
       "      <td>My homeboys tried to warn me, but __ makes me ...</td>\n",
       "      <td>Trimming the poop out of Chewbacca's butt hair</td>\n",
       "      <td>9</td>\n",
       "      <td>1989</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>My homeboys tried to warn me, but Trimming the...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957077</th>\n",
       "      <td>152124</td>\n",
       "      <td>Members of New York's social elite are paying ...</td>\n",
       "      <td>The vagina hole</td>\n",
       "      <td>4</td>\n",
       "      <td>1919</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Members of New York's social elite are paying ...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957078</th>\n",
       "      <td>215540</td>\n",
       "      <td>Why am I broke?</td>\n",
       "      <td>Injecting speed into one arm and horse tranqui...</td>\n",
       "      <td>8</td>\n",
       "      <td>1054</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>Why am I broke? Injecting speed into one arm a...</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957079</th>\n",
       "      <td>146987</td>\n",
       "      <td>What left this stain on my couch?</td>\n",
       "      <td>Whomsoever let the dogs out</td>\n",
       "      <td>3</td>\n",
       "      <td>2093</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>What left this stain on my couch? Whomsoever l...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1957080 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         fake_round_id                                    black_card_text  \\\n",
       "0               162435   Can a woman really have it all? A career and __?   \n",
       "1                76153  Well if you'll excuse me, gentlemen, I have a ...   \n",
       "2               184382  Here at the Academy for Gifted Children, we al...   \n",
       "3                57103  Armani suit: $1,000. Dinner for two at that sw...   \n",
       "4                99201  Hey, honey. For the bedroom, were you thinking...   \n",
       "...                ...                                                ...   \n",
       "1957075         218472  Come to Dubai, where you can relax in our worl...   \n",
       "1957076         197764  My homeboys tried to warn me, but __ makes me ...   \n",
       "1957077         152124  Members of New York's social elite are paying ...   \n",
       "1957078         215540                                    Why am I broke?   \n",
       "1957079         146987                  What left this stain on my couch?   \n",
       "\n",
       "                                           white_card_text  ID_index  \\\n",
       "0                               Bad bitches with fat asses         4   \n",
       "1               Seizing control of the means of production         4   \n",
       "2                                        The Devil himself         9   \n",
       "3                                          The vagina hole         5   \n",
       "4                            How awesome it is to be white         9   \n",
       "...                                                    ...       ...   \n",
       "1957075            New DNA evidence exonerating OJ Simpson        10   \n",
       "1957076     Trimming the poop out of Chewbacca's butt hair         9   \n",
       "1957077                                    The vagina hole         4   \n",
       "1957078  Injecting speed into one arm and horse tranqui...         8   \n",
       "1957079                        Whomsoever let the dogs out         3   \n",
       "\n",
       "         id_white  round_completion_seconds  won  \\\n",
       "0             392                        26    1   \n",
       "1            1536                        50    0   \n",
       "2            1739                        31    0   \n",
       "3            1919                         6    0   \n",
       "4            1015                        24    0   \n",
       "...           ...                       ...  ...   \n",
       "1957075      1315                        13    0   \n",
       "1957076      1989                        11    0   \n",
       "1957077      1919                         2    0   \n",
       "1957078      1054                        16    0   \n",
       "1957079      2093                        18    1   \n",
       "\n",
       "                                                      text  sum_won  \\\n",
       "0        Can a woman really have it all? A career and B...        3   \n",
       "1        Well if you'll excuse me, gentlemen, I have a ...        0   \n",
       "2        Here at the Academy for Gifted Children, we al...        0   \n",
       "3        Armani suit: $1,000. Dinner for two at that sw...        1   \n",
       "4        Hey, honey. For the bedroom, were you thinking...        1   \n",
       "...                                                    ...      ...   \n",
       "1957075  Come to Dubai, where you can relax in our worl...        0   \n",
       "1957076  My homeboys tried to warn me, but Trimming the...        0   \n",
       "1957077  Members of New York's social elite are paying ...        0   \n",
       "1957078  Why am I broke? Injecting speed into one arm a...        2   \n",
       "1957079  What left this stain on my couch? Whomsoever l...        1   \n",
       "\n",
       "         pair_count  \n",
       "0                 5  \n",
       "1                 3  \n",
       "2                 4  \n",
       "3                 6  \n",
       "4                 4  \n",
       "...             ...  \n",
       "1957075           1  \n",
       "1957076           4  \n",
       "1957077           2  \n",
       "1957078           5  \n",
       "1957079           4  \n",
       "\n",
       "[1957080 rows x 10 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_HurL_DrAK68",
    "outputId": "0c37e546-ad45-45f4-b36d-e494050e4141"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1957080, 384)\n",
      "(489280, 384)\n"
     ]
    }
   ],
   "source": [
    "y = df[\"won\"].values\n",
    "# X = np.concatenate([s1_emb,s2_emb],axis=1) # BOTH \n",
    "\n",
    "X =s1_emb\n",
    "X_test = s1_emb_test\n",
    "# try:\n",
    "#   X = s2_emb # BOTH \n",
    "#   X_test = s2_emb_test\n",
    "# except:\n",
    "#   X =s1_emb\n",
    "#   X_test = s1_emb_test\n",
    "# X = np.concatenate([s1_emb,vector_diffs],axis=1)## ALT - white cards + diffs\n",
    "\n",
    "print(X.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "AWxHrgvdt9kj"
   },
   "outputs": [],
   "source": [
    "# X = np.column_stack((X,output)) ### memory crash ? \n",
    "# X = np.column_stack((X,dot_sim)) ## dot product \n",
    "\n",
    "# # X = np.column_stack((X,output_2))\n",
    "# # X = np.column_stack((X,output_3))\n",
    "# X = np.column_stack((X,mean_diff))\n",
    "# X = np.column_stack((X,max_diff))\n",
    "# X = np.column_stack((X,min_diff))\n",
    "# print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "id": "NZWgLq44wVT7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "\n",
      "roc_auc 59.45\n",
      "proba based:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96    440352\n",
      "           1       1.00      0.18      0.30     48928\n",
      "\n",
      "    accuracy                           0.92    489280\n",
      "   macro avg       0.96      0.59      0.63    489280\n",
      "weighted avg       0.92      0.92      0.89    489280\n",
      "\n",
      "rocAUC: 0.5945\n",
      "top 1 acc by round: 17.69\n",
      "top 2 acc by round: 15.81\n",
      "top 3 acc by round: 14.59\n",
      "top 1 acc by round: 0.1769\n",
      "top 2 acc by round: 0.3163\n",
      "top 3 acc by round: 0.4376\n",
      "CPU times: total: 11min 3s\n",
      "Wall time: 11min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# StandardScaler\n",
    "# clf = make_pipeline(StandardScaler(),  LogisticRegression(solver=\"sag\")) ## PCA(n_components=2), # error with array? \n",
    "clf = LogisticRegression(solver=\"sag\")\n",
    "# clf = LogisticRegression() ## same res. ~ 3 min\n",
    "# cv_preds = cross_val_predict(clf,X,y,n_jobs=-2)\n",
    "\n",
    "clf.fit(X,y)\n",
    "y_test_preds = clf.predict(X_test)\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds))\n",
    "y_test_preds_proba = clf.predict_proba(X_test)[:,1]\n",
    "print(\"\\nroc_auc\",100*round(roc_auc_score(y_true=y_test,y_score=y_test_preds_proba),4))\n",
    "print(\"proba based:\")\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_proba>=0.5))\n",
    "\n",
    "eval_preds(df_test,y_test_preds_proba)\n",
    "### on sample - roc_auc 59.25 \n",
    "# \"\"\"\n",
    "# cv_preds = cross_val_predict(clf,X,y,method=\"predict_proba\",cv=3,n_jobs=-2)[:,1]\n",
    "# print(classification_report(y_true=y,y_pred=cv_preds>=0.5))\n",
    "# print(\"roc_auc\",roc_auc_score(y_true=y,y_score=cv_preds))\n",
    "\n",
    "\n",
    "# # #### min 4 (joint text), v1, 371K records\n",
    "# #               precision    recall  f1-score   support\n",
    "\n",
    "# #            0       0.65      0.90      0.75    227115\n",
    "# #            1       0.58      0.22      0.32    144299\n",
    "\n",
    "# #     accuracy                           0.64    371414\n",
    "# #    macro avg       0.61      0.56      0.54    371414\n",
    "# # weighted avg       0.62      0.64      0.58    371414\n",
    "\n",
    "# # roc_auc 0.620\n",
    "\n",
    "\n",
    "# \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "C571NK8Ozcqz"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "\n",
      "roc_auc 50.0\n",
      "proba based:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    440352\n",
      "           1       1.00      1.00      1.00     48928\n",
      "\n",
      "    accuracy                           1.00    489280\n",
      "   macro avg       1.00      1.00      1.00    489280\n",
      "weighted avg       1.00      1.00      1.00    489280\n",
      "\n",
      "rocAUC: 0.5\n",
      "top 1 acc by round: 10.03\n",
      "top 2 acc by round: 10.03\n",
      "top 3 acc by round: 9.99\n",
      "top 1 acc by round: 0.1003\n",
      "top 2 acc by round: 0.2006\n",
      "top 3 acc by round: 0.2996\n",
      "CPU times: total: 19min 35s\n",
      "Wall time: 3min 22s\n"
     ]
    }
   ],
   "source": [
    "# %%time\n",
    "# ## slow... ?  30 min\n",
    "# clf_rf = RandomForestClassifier(n_estimators=40,min_samples_split=3, min_samples_leaf=4,max_depth=16,\n",
    "#                                 n_jobs=-2, max_samples=0.25\n",
    "#                                 ,ccp_alpha=0.015) # n_jobs=-2\n",
    "\n",
    "# # cv_preds_2 = cross_val_predict(clf_rf,X,y,method=\"predict_proba\",cv=3)[:,1]\n",
    "# # print(classification_report(y_true=y,y_pred=cv_preds_2>=0.5))\n",
    "# # print(\"roc_auc\",roc_auc_score(y_true=y,y_score=cv_preds_2))\n",
    "\n",
    "# clf_rf.fit(X,y)\n",
    "# print(\"Trained\")\n",
    "# y_test_preds_rf = clf_rf.predict(X_test)\n",
    "# print(classification_report(y_true=y_test,y_pred=y_test_preds_rf))\n",
    "# y_test_preds_proba_rf = clf_rf.predict_proba(X_test)[:,1]\n",
    "# print(\"\\nroc_auc\",100*round(roc_auc_score(y_true=y_test,y_score=y_test_preds_proba_rf),4))\n",
    "# print(\"proba based:\")\n",
    "# print(classification_report(y_true=y_test,y_pred=y_test_preds_proba_rf>=0.5))\n",
    "\n",
    "# eval_preds(df_test,y_test_preds_proba_rf)\n",
    "# # # #### min 4 (joint text), v1, 371K records\n",
    "# #               precision    recall  f1-score   support\n",
    "\n",
    "# #            0       0.63      0.94      0.76    227115\n",
    "# #            1       0.61      0.14      0.23    144299\n",
    "\n",
    "# #     accuracy                           0.63    371414\n",
    "# #    macro avg       0.62      0.54      0.49    371414\n",
    "# # weighted avg       0.62      0.63      0.55    371414\n",
    "\n",
    "# # roc_auc 0.609\n",
    "\n",
    "\n",
    "\n",
    "# ##TODO - feature importance (do the deltas improve model?)\n",
    "\n",
    "# # min 2 , L12 miniLM\n",
    "# #               precision    recall  f1-score   support\n",
    "\n",
    "# #            0       0.73      0.97      0.83    469455\n",
    "# #            1       0.53      0.10      0.17    183561\n",
    "\n",
    "# #     accuracy                           0.72    653016\n",
    "# #    macro avg       0.63      0.53      0.50    653016\n",
    "# # weighted avg       0.68      0.72      0.65    653016\n",
    "\n",
    "# # roc_auc 0.64871\n",
    "# # CPU times: user 2h "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "G478iE9fuHSW"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.59      0.72    440352\n",
      "           1       0.12      0.50      0.20     48928\n",
      "\n",
      "    accuracy                           0.59    489280\n",
      "   macro avg       0.52      0.55      0.46    489280\n",
      "weighted avg       0.84      0.59      0.67    489280\n",
      "\n",
      "\n",
      "roc_auc 56.75\n",
      "proba based:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.59      0.72    440352\n",
      "           1       0.12      0.50      0.20     48928\n",
      "\n",
      "    accuracy                           0.59    489280\n",
      "   macro avg       0.52      0.55      0.46    489280\n",
      "weighted avg       0.84      0.59      0.67    489280\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.96    440352\n",
      "           1       1.00      0.16      0.27     48928\n",
      "\n",
      "    accuracy                           0.92    489280\n",
      "   macro avg       0.96      0.58      0.61    489280\n",
      "weighted avg       0.92      0.92      0.89    489280\n",
      "\n",
      "rocAUC: 0.5675\n",
      "top 1 acc by round: 15.86\n",
      "top 2 acc by round: 14.41\n",
      "top 3 acc by round: 13.42\n",
      "top 1 acc by round: 0.1586\n",
      "top 2 acc by round: 0.2881\n",
      "top 3 acc by round: 0.4025\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "clf_rf = RandomForestClassifier(n_estimators=40,min_samples_split=3, min_samples_leaf=4,max_depth=23,\n",
    "                                class_weight=\"balanced\",\n",
    "                                n_jobs=-2, max_samples=0.3\n",
    "                                ,ccp_alpha=0.0001) # n_jobs=-2\n",
    "\n",
    "# cv_preds_2 = cross_val_predict(clf_rf,X,y,method=\"predict_proba\",cv=3)[:,1]\n",
    "# print(classification_report(y_true=y,y_pred=cv_preds_2>=0.5))\n",
    "# print(\"roc_auc\",roc_auc_score(y_true=y,y_score=cv_preds_2))\n",
    "\n",
    "clf_rf.fit(X,y)\n",
    "print(\"Trained\")\n",
    "y_test_preds_rf = clf_rf.predict(X_test)\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_rf))\n",
    "y_test_preds_proba_rf = clf_rf.predict_proba(X_test)[:,1]\n",
    "print(\"\\nroc_auc\",100*round(roc_auc_score(y_true=y_test,y_score=y_test_preds_proba_rf),4))\n",
    "print(\"proba based:\")\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_proba_rf>=0.5))\n",
    "\n",
    "eval_preds(df_test,y_test_preds_proba_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "\n",
      "roc_auc 60.39\n",
      "proba based:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96    440352\n",
      "           1       1.00      0.18      0.31     48928\n",
      "\n",
      "    accuracy                           0.92    489280\n",
      "   macro avg       0.96      0.59      0.63    489280\n",
      "weighted avg       0.92      0.92      0.89    489280\n",
      "\n",
      "rocAUC: 0.6039\n",
      "top 1 acc by round: 18.03\n",
      "top 2 acc by round: 16.16\n",
      "top 3 acc by round: 14.82\n",
      "top 1 acc by round: 0.1803\n",
      "top 2 acc by round: 0.3232\n",
      "top 3 acc by round: 0.4446\n",
      "CPU times: total: 29min 48s\n",
      "Wall time: 5min 42s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "clf_rf = HistGradientBoostingClassifier() # n_jobs=-2\n",
    "\n",
    "# cv_preds_2 = cross_val_predict(clf_rf,X,y,method=\"predict_proba\",cv=3)[:,1]\n",
    "# print(classification_report(y_true=y,y_pred=cv_preds_2>=0.5))\n",
    "# print(\"roc_auc\",roc_auc_score(y_true=y,y_score=cv_preds_2))\n",
    "\n",
    "clf_rf.fit(X,y)\n",
    "print(\"Trained\")\n",
    "y_test_preds_rf = clf_rf.predict(X_test)\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_rf))\n",
    "y_test_preds_proba_rf = clf_rf.predict_proba(X_test)[:,1]\n",
    "print(\"\\nroc_auc\",100*round(roc_auc_score(y_true=y_test,y_score=y_test_preds_proba_rf),4))\n",
    "print(\"proba based:\")\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_proba_rf>=0.5))\n",
    "\n",
    "eval_preds(df_test,y_test_preds_proba_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WIEDabG88ax8"
   },
   "outputs": [],
   "source": [
    "df[\"text\"].str.split().str.len().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RVhmVMGq3Vvi"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "# y = df[\"picks\"].values\n",
    "# # X = np.concatenate([s1_emb,s2_emb],axis=1) # BOTH \n",
    "# X = list(df[\"black_card_text\"].values)\n",
    "\"\"\"\n",
    "##ORIG:\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    list(df[\"text\"].values), list(df[\"won\"].values), test_size=0.25, random_state=42)\n",
    "\"\"\"\n",
    "## NEW:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1WEZ5crL80SI"
   },
   "outputs": [],
   "source": [
    "set(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qPs1eIc14Ei0"
   },
   "outputs": [],
   "source": [
    "# Try finetuning? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pdKgjj3tzcx4"
   },
   "outputs": [],
   "source": [
    "### https://www.sbert.net/docs/training/overview.html\n",
    "from torch import nn\n",
    "\n",
    "# word_embedding_model = models.Transformer('bert-base-uncased', max_seq_length=256)\n",
    "# pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())\n",
    "dense_model = models.Dense(in_features=pooling_model.get_sentence_embedding_dimension(), out_features=128, activation_function=nn.Tanh())\n",
    "\n",
    "# model2 = SentenceTransformer(modules=[word_embedding_model, pooling_model, dense_model])\n",
    "model2 = SentenceTransformer(modules=[model, dense_model])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_5gL5gVG8rAt"
   },
   "outputs": [],
   "source": [
    "len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9lZuN3eO5Ex7"
   },
   "outputs": [],
   "source": [
    "# [X_train[0:5],y_train[0:5]]\n",
    "## list(df[\"black_card_text\"].values)\n",
    "train_examples =  [InputExample(texts=[X_train[i]],label=float(y_train[i])) for i in range(len(X_train)-1)]\n",
    "# [X_train[0:5],y_train[0:5]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MqwId_8b7zel"
   },
   "outputs": [],
   "source": [
    "range(len(X_train)-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jC_4K4HK7vDg"
   },
   "outputs": [],
   "source": [
    "X_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eAQC1-uI7sA3"
   },
   "outputs": [],
   "source": [
    "len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2o8dxHLV7osN"
   },
   "outputs": [],
   "source": [
    "len(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JKBO4Rj67lsZ"
   },
   "outputs": [],
   "source": [
    "len(train_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "smasDpEz38-E"
   },
   "outputs": [],
   "source": [
    "### IndexError: list index out of range. ? \n",
    "\n",
    "\n",
    "#Define your train examples. You need more than just two examples...\n",
    "# train_examples = [InputExample(texts=['My first sentence', 'My second sentence'], label=0.8),\n",
    "#     InputExample(texts=['Another pair', 'Unrelated sentence'], label=0.3)]\n",
    "\n",
    "# train_examples =  [X_train[0:5],y_train[0:5]]\n",
    "\n",
    "#Define your train dataset, the dataloader and the train loss\n",
    "train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=64)\n",
    "# train_loss = losses.CosineSimilarityLoss(model)\n",
    "\n",
    "# #Tune the model\n",
    "# model2.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "izHedCfQ985E"
   },
   "outputs": [],
   "source": [
    "### https://www.sbert.net/docs/package_reference/losses.html#batchsemihardtripletloss\n",
    "## supports 1 sentence/label pair\n",
    "train_dataset = SentencesDataset(train_examples, model2)\n",
    "train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=256)\n",
    "train_loss = losses.BatchSemiHardTripletLoss(model=model2)\n",
    "\n",
    "\n",
    "#Tune the model\n",
    "model2.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uxfcIQ85-nlQ"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZrImtf8s-mJ8"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "74QYHj-1mljC"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of 3-CAH_finetune.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "030325446b8e4ede91020a4536f91064": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0b2804c196ef4f369aa469a3080cb930": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b17a4f8b6fca414286b47a65e60d1254",
       "IPY_MODEL_6b8fcd36f1bb4a22981f0162ae538d7f",
       "IPY_MODEL_62bb978e5a3b4a9cab47a8856c3633d0"
      ],
      "layout": "IPY_MODEL_2941cf6f1bf745b7a8518ce2b1e8aaf7"
     }
    },
    "12f0eead030b48e79d989ab53ae75f5a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "24c0db6a9c2146bea1b658c7245044aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ef2b365a6ece4bdb8d9cb84b6e593047",
      "max": 9786,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_35d4592d40f344ae92e27bd0bdc0f129",
      "value": 9786
     }
    },
    "2941cf6f1bf745b7a8518ce2b1e8aaf7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "35d4592d40f344ae92e27bd0bdc0f129": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "3f969476da554298b79fb91c4ec6e08f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9952f2e5f62d402eadd85c0c17d5d8c2",
      "placeholder": "​",
      "style": "IPY_MODEL_12f0eead030b48e79d989ab53ae75f5a",
      "value": "Batches: 100%"
     }
    },
    "62bb978e5a3b4a9cab47a8856c3633d0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b597b302d1d9405fbf9181d8a62e3944",
      "placeholder": "​",
      "style": "IPY_MODEL_a5e9540e89ec4bee85fb45235c2fc640",
      "value": " 2447/2447 [05:01&lt;00:00, 21.96it/s]"
     }
    },
    "6b8fcd36f1bb4a22981f0162ae538d7f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8c44dbe9d1c74aae894b1ff86ae69c9b",
      "max": 2447,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_75d0eb084e1b429b9375196c6069c55d",
      "value": 2447
     }
    },
    "75d0eb084e1b429b9375196c6069c55d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "82841c43fda44080b6debfd95e425b0f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d54fadb082c64e1e868ed9b21bdb4698",
      "placeholder": "​",
      "style": "IPY_MODEL_9933a11516d64e00a16d8793fbb2ac04",
      "value": " 9786/9786 [19:57&lt;00:00, 22.67it/s]"
     }
    },
    "82cc1edd77f047b084a5f0c0b17a4bb1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8c44dbe9d1c74aae894b1ff86ae69c9b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9933a11516d64e00a16d8793fbb2ac04": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9952f2e5f62d402eadd85c0c17d5d8c2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9f284c9e3c364f778c41b8c7e0ba3045": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a5e9540e89ec4bee85fb45235c2fc640": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b17a4f8b6fca414286b47a65e60d1254": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9f284c9e3c364f778c41b8c7e0ba3045",
      "placeholder": "​",
      "style": "IPY_MODEL_030325446b8e4ede91020a4536f91064",
      "value": "Batches: 100%"
     }
    },
    "b597b302d1d9405fbf9181d8a62e3944": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cfea03270fe64a44b54cde507f629eaf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3f969476da554298b79fb91c4ec6e08f",
       "IPY_MODEL_24c0db6a9c2146bea1b658c7245044aa",
       "IPY_MODEL_82841c43fda44080b6debfd95e425b0f"
      ],
      "layout": "IPY_MODEL_82cc1edd77f047b084a5f0c0b17a4bb1"
     }
    },
    "d54fadb082c64e1e868ed9b21bdb4698": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ef2b365a6ece4bdb8d9cb84b6e593047": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}