albert-Copy_3_CAH_finetune_DeepLearning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Use paraphrase-albert-small-v2 instead"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "liIk8f980xT5"
   },
   "source": [
    "# Sentence Embeddings using Siamese BERT-Networks/Sentence Transformers\n",
    "* Also TSDAE. pretraining? \n",
    "---\n",
    "\n",
    "The Sentence Transformer library is available on [pypi](https://pypi.org/project/sentence-transformers/) and [github](https://github.com/UKPLab/sentence-transformers). The library implements code from the ACL 2019 paper entitled \"[Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410.pdf)\" by Nils Reimers and Iryna Gurevych.\n",
    "\n",
    "* https://github.com/UKPLab/sentence-transformers/blob/master/examples/unsupervised_learning/TSDAE/train_askubuntu_tsdae.py\n",
    "* https://www.sbert.net/examples/unsupervised_learning/TSDAE/README.html#tsdae-as-pre-training-task\n",
    "* https://www.sbert.net/docs/quickstart.html\n",
    "\n",
    "This version also combined white/black pairs and trains over a single column/text\n",
    "* https://datascience.stackexchange.com/questions/39345/how-to-replace-a-part-string-value-of-a-column-using-another-column\n",
    "* NOTE: Some cards lack a \"____\" - need to handle them \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FIjj9BjkY_A9",
    "outputId": "77d91f42-194e-4049-9a4c-7dd293569829"
   },
   "outputs": [],
   "source": [
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bzljhyTQEZds"
   },
   "source": [
    "## Install Sentence Transformer Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AmxRYxNDvn6y",
    "outputId": "023199f5-52cf-4a9e-80cb-cd3b60e58969"
   },
   "outputs": [],
   "source": [
    "# # Install the library using pip\n",
    "# !pip3 install sentence-transformers scikit-learn tensorflow -U\n",
    "# # !pip3 install sentence-transformers scikit-learn tensorflow tensorflow-text tf-models-official -U\n",
    "# # !pip3 install scikit-learn  -U\n",
    "# # !pip3 install nltk scikit-learn  -U\n",
    "\n",
    "# # import nltk\n",
    "# # nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "jS9O0eAwlt8U"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n",
    "from sklearn.model_selection import train_test_split,GridSearchCV, cross_val_score, cross_val_predict\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.metrics import classification_report, roc_auc_score  \n",
    "\n",
    "# https://stackoverflow.com/questions/53784971/how-to-disable-convergencewarning-using-sklearn\n",
    "from warnings import simplefilter\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "simplefilter(\"ignore\", category=ConvergenceWarning)\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "FcJ95uWKXoJC"
   },
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, LoggingHandler\n",
    "from sentence_transformers import models, util, datasets, evaluation, losses, SentencesDataset ## https://www.sbert.net/docs/package_reference/losses.html + SentencesDataset\n",
    "from sentence_transformers import SentenceTransformer, InputExample, losses\n",
    "from torch.utils.data import DataLoader\n",
    "from sentence_transformers import SentenceTransformer, SentencesDataset, losses ## MultipleNegativesRankingLoss\n",
    "## https://www.sbert.net/docs/package_reference/losses.html#multiplenegativesrankingloss\n",
    "# from sentence_transformers.readers import InputExample\n",
    "import logging\n",
    "import os\n",
    "import gzip\n",
    "from torch.utils.data import DataLoader\n",
    "from datetime import datetime\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "1gRQUdycC9aR"
   },
   "outputs": [],
   "source": [
    "model_name = \"paraphrase-albert-small-v2\"#\"all-MiniLM-L12-v2\" #\"all-MiniLM-L6-v2\" # \"all-MiniLM-L12-v2\"\n",
    "# model_name = \"/content/drive/MyDrive/research/cah/cah_tsdae-model\" # prev\n",
    "min_cooccurences = 1 #4 # filter sentences for pairs that occurred at least k times. min 5: 200K. min 1: 1.9M\n",
    "\n",
    "ONE_COL_DATA_FORMAT = True#False\n",
    "USE_TEXT_COLS =  [\"text\"]#[\"text\",\t\"white_card_text\"]#[\"black_card_text\",\t\"white_card_text\"]\n",
    "\n",
    "FAST_RUN = False#True#False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eIAKz6KVEndZ"
   },
   "source": [
    "## Load the sBERT Model\n",
    "\n",
    "* Default , later try pretrained+ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "5IO_j2Ofv5pq"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50ebdea40249498aa26029e37a03f076",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/690 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa5af3bd40fa4166a57332b5decc992a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/190 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4b98a61c963413b91a822b3817ecd3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/4.03k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ae30544a2ae41dd918fab43c7d0467e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/827 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5ed22f5305b4d1fb2d44cb5fbff51f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/122 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7ec0a0e741940118c335abbaf0347b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/229 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "beae83ce4a8f4a8a9b44c6d2c5c074a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/46.7M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37e41e0fef2e44d4b2a3dab3a0d18d11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29c42bbcfe0d4cefa13db578606e6878",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/245 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6668718ca6514ee98327fe1255e260bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/760k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8667dec76d2e4fcdbab0a534009e359c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.31M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de2d6e474fe643ea8f4c212a03f17353",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/465 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load the BERT model. Various models trained on Natural Language Inference (NLI) https://github.com/UKPLab/sentence-transformers/blob/master/docs/pretrained-models/nli-models.md and \n",
    "# Semantic Textual Similarity are available https://github.com/UKPLab/sentence-transformers/blob/master/docs/pretrained-models/sts-models.md\n",
    "\n",
    "# model = SentenceTransformer('bert-base-nli-mean-tokens')\n",
    "# model = SentenceTransformer(\"nli-distilroberta-base-v2\")\n",
    "## # https://www.sbert.net/docs/pretrained_models.html\n",
    "# model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\") ## \"paraphrase-MiniLM-L12-v2\"\n",
    "# model = SentenceTransformer(\"paraphrase-MiniLM-L12-v2\")\n",
    "model = SentenceTransformer(model_name)\n",
    "# model = SentenceTransformer(\"all-MiniLM-L12-v2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "1b9dorxmsWtJ"
   },
   "outputs": [],
   "source": [
    "## copied from another notebook; may not be most up to date??\n",
    "\n",
    "def eval_preds(df_test,preds):\n",
    "    df2 = df_test[[\"fake_round_id\",\"won\"]].copy()\n",
    "    df2[\"preds\"] = preds\n",
    "    df2 = df2.sample(frac=1)\n",
    "    df2[\"m_score\"] = df2.groupby(\"fake_round_id\")[\"preds\"].transform(\"max\")\n",
    "    df2[\"correct\"] = ((df2[\"preds\"]==df2[\"m_score\"]) &(df2[\"won\"]>0)).astype(int)#df2.loc[df2[\"preds\"]==df2[\"m_score\"]\n",
    "    # df2.tail(31) ## there are cases wtih multiple values with same score/rank?  - ignore for now\n",
    "    df2.sort_values(\"preds\",ascending=False,inplace=True)\n",
    "    print(classification_report(df2[\"won\"],df2[\"correct\"]))\n",
    "    print(\"rocAUC:\",round(roc_auc_score(df2[\"won\"],df2[\"preds\"]),4))\n",
    "    print(f'top 1 acc by round: {100*df2.groupby(\"fake_round_id\").head(1)[\"won\"].mean():.2f}') # older ?\n",
    "    print(f'top 2 acc by round: {100*df2.groupby(\"fake_round_id\").head(2)[\"won\"].mean():.2f}')# older ?\n",
    "    print(f'top 3 acc by round: {100*df2.groupby(\"fake_round_id\").head(3)[\"won\"].mean():.2f}')# older ?\n",
    "    print(\"top 1 acc by round:\",round(df2.groupby(\"fake_round_id\").head(1)[\"won\"].mean(),4))\n",
    "    print(\"top 2 acc by round:\",round(df2.groupby(\"fake_round_id\").head(2).groupby(\"fake_round_id\")[\"won\"].max().mean(),4))\n",
    "    print(\"top 3 acc by round:\",round(df2.groupby(\"fake_round_id\").head(3).groupby(\"fake_round_id\")[\"won\"].max().mean(),4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I7_Ib3ITEwgO"
   },
   "source": [
    "## Setup a Corpus\n",
    "\n",
    "* `/content/drive/MyDrive/Research/CAH/cah_train_min7_v1.csv` - 37K\n",
    "    * `/content/drive/MyDrive/Research/CAH/cah_min7_v2.csv.gz`\n",
    "* `/content/drive/MyDrive/Research/CAH/cah_train_min6_v1.csv`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "gT9VrZp1_mvO"
   },
   "outputs": [],
   "source": [
    "# KEEP_COLS = [\"black_card_text\",\"white_card_text\",\"text\",\"picks\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "K7tGxkF4l-oB",
    "outputId": "4c2637b9-8d86-4e51-e7c1-44f937749403"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1957080 rows\n",
      "             won  pair_count\n",
      "count  1957080.0  1957080.00\n",
      "mean         0.1        3.53\n",
      "std          0.3        1.60\n",
      "min          0.0        1.00\n",
      "25%          0.0        2.00\n",
      "50%          0.0        3.00\n",
      "75%          0.0        4.00\n",
      "max          1.0       13.00\n",
      "1957080\n"
     ]
    }
   ],
   "source": [
    "## \"/content/drive/MyDrive/Research/CAH/cah_min6_v2.csv.gz\"\n",
    "# \"/content/drive/MyDrive/Research/CAH/cah_min2_v2.csv.gz\"\n",
    "# \"/content/drive/MyDrive/Research/CAH/cah_train_min4_v1.csv\"\n",
    "# df = pd.read_csv(\"/content/drive/MyDrive/Research/CAH/cah_min6_v2.csv.gz\",usecols=[\"black_card_text\",\"white_card_text\",\"picks\"])#.sample(99)\n",
    "# df = pd.read_csv(\"/content/drive/MyDrive/Research/CAH/cah_train_min4_v1.csv\",usecols=[\"black_card_text\",\"white_card_text\",\"picks\"])#.sample(99)\n",
    "\n",
    "# df = pd.read_parquet(\"/content/drive/MyDrive/research/cah/cah_train_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\") \n",
    "df = pd.read_parquet(\"cah_train_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\") \n",
    "\n",
    "if FAST_RUN:\n",
    "    df = df.sort_values(\"fake_round_id\").head(11230)\n",
    "df[\"won\"] = df[\"won\"].astype(int)\n",
    "df = df.sort_values(by=\"won\",ascending=False) ## get picked pairs first\n",
    "\n",
    "# df_all = df.copy() ## copy for quick eval\n",
    "print(df.shape[0],\"rows\")\n",
    "\n",
    "df[\"pair_count\"] = df.groupby(\"text\")[\"won\"].transform(\"count\") ## can be used to filter sentences occurring less than k times\n",
    "print(df[[\"won\",\"pair_count\"]].describe().round(2) )\n",
    "\n",
    "# df = df.drop_duplicates(subset=[\"black_card_text\",\"white_card_text\"],keep=\"first\")#.sample(frac=1)\n",
    "print(df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# df_embeds = pd.read_parquet(\"cah_embed_L12.parquet\") ## existing file I have locally. is it good? \n",
    "# df_embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_embeds.isna().max(axis=0).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vyWPiyqQBphJ"
   },
   "source": [
    "Test set\n",
    "* Keeps round level grouping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 728
    },
    "id": "IxyngBX8ACCc",
    "outputId": "554ba40b-2191-4469-9459-3a4e40c21bf6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fake_round_id                48928\n",
      "black_card_text                580\n",
      "white_card_text               2118\n",
      "ID_index                        10\n",
      "id_white                      2118\n",
      "round_completion_seconds       705\n",
      "won                              2\n",
      "text                        377397\n",
      "sum_won                          9\n",
      "dtype: int64\n",
      "489280 rows\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fake_round_id</th>\n",
       "      <th>black_card_text</th>\n",
       "      <th>white_card_text</th>\n",
       "      <th>ID_index</th>\n",
       "      <th>id_white</th>\n",
       "      <th>round_completion_seconds</th>\n",
       "      <th>won</th>\n",
       "      <th>text</th>\n",
       "      <th>sum_won</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2387657</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>The NRA</td>\n",
       "      <td>8</td>\n",
       "      <td>1755</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387651</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Fisting</td>\n",
       "      <td>2</td>\n",
       "      <td>790</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387653</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Crab</td>\n",
       "      <td>4</td>\n",
       "      <td>590</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387654</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Slowly releasing a huge fart over the course o...</td>\n",
       "      <td>5</td>\n",
       "      <td>1591</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387655</th>\n",
       "      <td>238766</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>Elon Musk</td>\n",
       "      <td>6</td>\n",
       "      <td>721</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>What's about to take this dance floor to the n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989543</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Holding up the line at Walgreens by trying to ...</td>\n",
       "      <td>4</td>\n",
       "      <td>997</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! Holding up the line at Walgreens by...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989545</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Breastfeeding a ten-year-old</td>\n",
       "      <td>6</td>\n",
       "      <td>495</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! Breastfeeding a ten-year-old killed...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989542</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>A bird that shits human turds</td>\n",
       "      <td>3</td>\n",
       "      <td>56</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! A bird that shits human turds kille...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989547</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Jerking off to a 10-second RealMedia clip</td>\n",
       "      <td>8</td>\n",
       "      <td>1077</td>\n",
       "      <td>7613</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! Jerking off to a 10-second RealMedi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989541</th>\n",
       "      <td>298955</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Jeff Bezos</td>\n",
       "      <td>2</td>\n",
       "      <td>1073</td>\n",
       "      <td>7613</td>\n",
       "      <td>1</td>\n",
       "      <td>Oh my god! Jeff Bezos killed Kenny!</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>489280 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         fake_round_id                                    black_card_text  \\\n",
       "2387657         238766  What's about to take this dance floor to the n...   \n",
       "2387651         238766  What's about to take this dance floor to the n...   \n",
       "2387653         238766  What's about to take this dance floor to the n...   \n",
       "2387654         238766  What's about to take this dance floor to the n...   \n",
       "2387655         238766  What's about to take this dance floor to the n...   \n",
       "...                ...                                                ...   \n",
       "2989543         298955                        Oh my god! __ killed Kenny!   \n",
       "2989545         298955                        Oh my god! __ killed Kenny!   \n",
       "2989542         298955                        Oh my god! __ killed Kenny!   \n",
       "2989547         298955                        Oh my god! __ killed Kenny!   \n",
       "2989541         298955                        Oh my god! __ killed Kenny!   \n",
       "\n",
       "                                           white_card_text  ID_index  \\\n",
       "2387657                                            The NRA         8   \n",
       "2387651                                            Fisting         2   \n",
       "2387653                                               Crab         4   \n",
       "2387654  Slowly releasing a huge fart over the course o...         5   \n",
       "2387655                                          Elon Musk         6   \n",
       "...                                                    ...       ...   \n",
       "2989543  Holding up the line at Walgreens by trying to ...         4   \n",
       "2989545                       Breastfeeding a ten-year-old         6   \n",
       "2989542                      A bird that shits human turds         3   \n",
       "2989547          Jerking off to a 10-second RealMedia clip         8   \n",
       "2989541                                         Jeff Bezos         2   \n",
       "\n",
       "         id_white  round_completion_seconds  won  \\\n",
       "2387657      1755                        46    0   \n",
       "2387651       790                        46    0   \n",
       "2387653       590                        46    1   \n",
       "2387654      1591                        46    0   \n",
       "2387655       721                        46    0   \n",
       "...           ...                       ...  ...   \n",
       "2989543       997                      7613    0   \n",
       "2989545       495                      7613    0   \n",
       "2989542        56                      7613    0   \n",
       "2989547      1077                      7613    0   \n",
       "2989541      1073                      7613    1   \n",
       "\n",
       "                                                      text  sum_won  \n",
       "2387657  What's about to take this dance floor to the n...        1  \n",
       "2387651  What's about to take this dance floor to the n...        0  \n",
       "2387653  What's about to take this dance floor to the n...        1  \n",
       "2387654  What's about to take this dance floor to the n...        1  \n",
       "2387655  What's about to take this dance floor to the n...        0  \n",
       "...                                                    ...      ...  \n",
       "2989543  Oh my god! Holding up the line at Walgreens by...        0  \n",
       "2989545  Oh my god! Breastfeeding a ten-year-old killed...        0  \n",
       "2989542  Oh my god! A bird that shits human turds kille...        0  \n",
       "2989547  Oh my god! Jerking off to a 10-second RealMedi...        0  \n",
       "2989541                Oh my god! Jeff Bezos killed Kenny!        3  \n",
       "\n",
       "[489280 rows x 9 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_test = pd.read_parquet(\"/content/drive/MyDrive/research/cah/cah_test_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\").sample(frac=1)\n",
    "df_test = pd.read_parquet(\"cah_test_games.parquet\").drop(columns=[\"prior_white\"],errors=\"ignore\")\n",
    "df_test[\"won\"] = df_test[\"won\"].astype(int)\n",
    "\n",
    "if FAST_RUN:\n",
    "    df_test = df_test.sort_values(\"fake_round_id\").head(230)\n",
    "print(df_test.nunique())\n",
    "print(df_test.shape[0],\"rows\")\n",
    "df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kUh_gOK32mZu"
   },
   "source": [
    "### Mean baseline priors\n",
    "* By min cooccurrences of sentence pairs in in raw data\n",
    "```\n",
    "  1 min occ Prior Acc: 0.2044\n",
    "  2 min occ Prior Acc: 0.2032\n",
    "  3 min occ Prior Acc: 0.2027\n",
    "  4 min occ Prior Acc: 0.2011\n",
    "  5 min occ Prior Acc: 0.1922\n",
    "  6 min occ Prior Acc: 0.1762\n",
    "  7 min occ Prior Acc: 0.1503\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "QRC7aCEE2mBe"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "  for i in range(1,8):\n",
    "    print(f\"{i} min occ, {df_temp.shape[0]} rows\")\n",
    "    df_temp = df.loc[df[\"pair_count\"] >=i].copy()\n",
    "    df_white_prior = df_temp.groupby([\"white_card_text\"], as_index=False)[\"won\"].mean().rename(columns={\"won\":\"white_prior\"}).set_index(\"white_card_text\")\n",
    "    df_test = df_test.join(df_white_prior,on=\"white_card_text\",how=\"left\")\n",
    "    prior = df_test[\"white_prior\"].mean()\n",
    "    df_test[\"white_prior\"] = df_test[\"white_prior\"].fillna(prior)\n",
    "    print(\"White Prior Acc: %.3f\" %df_test.sort_values(\"white_prior\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    \n",
    "  ## prior for a black-white combination - mean (freq% won), or sum (times won?) , or threshholded max? (over 2 times?)\n",
    "    df_pair_prior = df_temp.groupby([\"white_card_text\",\"black_card_text\"], as_index=False)[\"won\"].sum().rename(columns={\"won\":\"pair_prior\"}).set_index([\"white_card_text\",\"black_card_text\"])\n",
    "    \n",
    "    df_test = df_test.join(df_pair_prior,on=[\"white_card_text\",\"black_card_text\"],how=\"left\")\n",
    "    prior = df_test[\"pair_prior\"].mean()\n",
    "    df_test[\"pair_prior\"] = df_test[\"pair_prior\"].fillna(prior)\n",
    "    print(\"Pair Prior (Only)  Acc: %.3f\" %df_test.sort_values(\"pair_prior\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    print(\"White then Pair Prior Acc: %.3f\" %df_test.sort_values([\"white_prior\",\"pair_prior\"],ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    print(\"Pair Prior then White Acc: %.3f\" %df_test.sort_values([\"pair_prior\",\"white_prior\",],ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.6% (with min 6) , 17.6% with min 3\n",
    "    \n",
    "    df_test.drop(columns=[\"white_prior\",\"pair_prior\"],errors=\"ignore\",inplace=True)\n",
    "except:()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zhvJzhL2BeFM"
   },
   "source": [
    "Drop duplicate instances with same out put (i.e ignore round level/ranking) \n",
    "* keep positives preferrably\n",
    "* Could do : weight or filter bby # occurrences\n",
    "\n",
    "* 22% mean win rate after this (instea of 10%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "OKAwvWRzBctu"
   },
   "outputs": [],
   "source": [
    "if min_cooccurences>1:\n",
    "  ## filter for sentence pairs occurring at least X times, regardless of label\n",
    "  df = df.loc[df[\"pair_count\"] >=min_cooccurences]\n",
    "  print(df.nunique())\n",
    "  print(df.shape[0],f\"rows after {min_cooccurences} filter of pairs\")\n",
    "  df = df.drop(columns=[\"fake_round_id\",\"prior_white\",\"pair_count\"],errors=\"ignore\") # drop round id if not doing group level ranking\n",
    "  try:\n",
    "    df = df.sort_values(by=\"won\",ascending=False).drop_duplicates(subset=[\"black_card_text\",\"white_card_text\"],keep=\"first\")\n",
    "    df_test = df_test.sort_values(by=\"won\",ascending=False).drop_duplicates(subset=[\"black_card_text\",\"white_card_text\"],keep=\"first\")\n",
    "  except: print(\"sort by and keep first failed\")\n",
    "\n",
    "  print(\"mean won:\",df[\"won\"].mean())\n",
    "  print(df.nunique())\n",
    "  print(df.shape[0],\"rows\")\n",
    "  df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "_1Wx14J-6uV1"
   },
   "outputs": [],
   "source": [
    "df = df.sample(frac=1)\n",
    "df_test = df_test.sample(frac=1)\n",
    "\n",
    "df.reset_index(inplace=True,drop=True)\n",
    "df_test.reset_index(inplace=True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "szXA3q0VWDjg"
   },
   "outputs": [],
   "source": [
    "# import csv\n",
    "# df[[#'black_card_text',\n",
    "#     # 'white_card_text',\n",
    "#   'text','won']].rename(columns={'won':\"target\"}).to_csv(f\"cah_min{min_cooccurences}.csv.gz\",compression=\"gzip\",index=False,quoting=csv.QUOTE_ALL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JUVdkcJ7hFu3"
   },
   "source": [
    "#### train - eval split (if doing supervised pretraining... )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PLDoriAMFDRE"
   },
   "source": [
    "##### Data formats: \n",
    "1. 2 text cols\n",
    "  * Can be white, black\n",
    "  * could be merged (`text`) and white/black,\n",
    "2. single joint text col\n",
    "  * Merged (`text`) col\n",
    "\n",
    "\n",
    "\n",
    "  https://www.pinecone.io/learn/train-sentence-transformers-softmax/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "9k8qHhym2x2w"
   },
   "outputs": [],
   "source": [
    "y = df[\"won\"].values\n",
    "\n",
    "if ONE_COL_DATA_FORMAT:\n",
    "  ### 1 text col version\n",
    "  ## orig:\n",
    "  # X_train, X_test, y_train, y_test = train_test_split(\n",
    "  #   list(df[\"text\"].values), list(df[\"won\"].astype(int).values), test_size=0.2, random_state=42)\n",
    "## NEW/ALT - use defined train, test:\n",
    "  X_train=list(df[\"text\"].values)\n",
    "  y_train = list(df[\"won\"].values)\n",
    "  X_test=list(df_test[\"text\"].values)\n",
    "  y_test = list(df_test[\"won\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "Msv2dCOphFIv"
   },
   "outputs": [],
   "source": [
    "# # X = np.concatenate([s1_emb,s2_emb],axis=1) # BOTH \n",
    "# # X = list(df[\"text\"].values)\n",
    "\n",
    "# if ONE_COL_DATA_FORMAT:\n",
    "#   ### 1 text col version\n",
    "#   ## orig:\n",
    "#   # X_train, X_test, y_train, y_test = train_test_split(\n",
    "#   #   list(df[\"text\"].values), list(df[\"won\"].astype(int).values), test_size=0.2, random_state=42)\n",
    "# ## NEW/ALT - use defined train, test:\n",
    "#   X_train=list(df[\"text\"].values)\n",
    "#   y_train = list(df[\"won\"].values)\n",
    "#   X_test=list(df_test[\"text\"].values)\n",
    "#   y_test = list(df_test[\"won\"].values)\n",
    "  \n",
    "#   train_samples=  [InputExample(texts=[X_train[i]],label=float(y_train[i])) for i in range(len(X_train))]\n",
    "#   test_samples=  [InputExample(texts=[X_test[i]],label=float(y_test[i])) for i in range(len(X_test))]\n",
    "\n",
    "#   ## added: copied from 2 text - expects 2 cols??\n",
    "#   ## LabelAccuracyEvaluator\n",
    "#   dev_eval = evaluation.BinaryClassificationEvaluator(sentences1=[x for x in X_test],sentences2 = [x for x in X_test],labels = y_test,\n",
    "#                                                                           batch_size=128,show_progress_bar=True,write_csv=True)\n",
    "# else:\n",
    "#   ### 2 text col version ; can try different cols\n",
    "#   \"\"\"\n",
    "#   ##ORIG:\n",
    "#   X_train, X_test, y_train, y_test = train_test_split(\n",
    "#     list(df[USE_TEXT_COLS].values), list(df[\"won\"].values), test_size=0.2, random_state=42)\n",
    "#   \"\"\"\n",
    "#   ## NEW/ALT - use defined train, test:\n",
    "#   X_train=list(df[USE_TEXT_COLS].values)\n",
    "#   y_train = list(df[\"won\"].values)\n",
    "#   X_test=list(df_test[USE_TEXT_COLS].values)\n",
    "#   y_test = list(df_test[\"won\"].values)\n",
    "#   train_samples=  [InputExample(texts=[X_train[i][0],X_train[i][1]],label=float(y_train[i])) for i in range(len(X_train))]\n",
    "#   test_samples=  [InputExample(texts=[X_test[i][0],X_test[i][1]],label=float(y_test[i])) for i in range(len(X_test))]\n",
    "#   ### binary evaluator: expect list 1, list 2:\n",
    "#   ## https://www.sbert.net/docs/package_reference/evaluation.html\n",
    "#   ## leaky: \n",
    "#   dev_eval = evaluation.BinaryClassificationEvaluator(sentences1=[x[0] for x in X_test],sentences2 = [x[1] for x in X_test],labels = y_test,\n",
    "#                                                                           batch_size=128,show_progress_bar=True,write_csv=True)\n",
    "# # train_examples =  [InputExample(texts=[X_train[i]],label=float(y_train[i])) for i in range(len(X_train))]\n",
    "# train_examples = X_train\n",
    "# test_examples = X_test # added"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ck8HAVmMZRli"
   },
   "source": [
    "* See how well unsupervised model does? \n",
    "* Check if funny combs are close or far"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "GO_PRyRAhVN6"
   },
   "outputs": [],
   "source": [
    "# model2 = SentenceTransformer(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "fZpw55M5ZaN2"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### could speed this up n^2 by getting embeddings only for unique combniations, then rejoining/merging...  (Won't work if using \"Text\" col)\n",
    "# s1_emb = model.encode(list(df[USE_TEXT_COLS[0]].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=128\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )\n",
    "\n",
    "# ## black cards = quesiton/prompt\n",
    "# s2_emb = model.encode(list(df[USE_TEXT_COLS[1]].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=256\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n",
    "\n",
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0]))\n",
    "# print(len(dot_sim))\n",
    "\n",
    "# df[\"dot_sim_score\"] = dot_sim\n",
    "# # df_all[\"dot_sim_score\"] = dot_sim\n",
    "# print(df[[\"dot_sim_score\",\"won\"]].corr())\n",
    "# print(\"df:\\n\",df.groupby(\"won\")[\"dot_sim_score\"].mean())\n",
    "\n",
    "# # print(\"df_All:\")\n",
    "# # print(df_all[[\"dot_sim_score\",\"won\"]].corr())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "uMXZ6ctoh-2-"
   },
   "outputs": [],
   "source": [
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0]))\n",
    "# print(len(dot_sim))\n",
    "\n",
    "# df[\"dot_sim_score\"] = dot_sim\n",
    "# # df_all[\"dot_sim_score\"] = dot_sim\n",
    "# print(df[[\"dot_sim_score\",\"won\"]].corr())\n",
    "# print(\"df:\\n\",df.groupby(\"won\")[\"dot_sim_score\"].mean())\n",
    "# print(\"df_all:\\n\",df_all.groupby(\"won\")[\"dot_sim_score\"].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OHa6wGqNghi5"
   },
   "source": [
    "### try supervised model\n",
    "\n",
    "* Contrastive loss?\n",
    "* https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/quora_duplicate_questions/training_OnlineContrastiveLoss.py\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html#training\n",
    "\n",
    "\n",
    "Can combine multiple losses:\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "o7Gzd4DmkB0N"
   },
   "outputs": [],
   "source": [
    "#As distance metric, we use cosine distance (cosine_distance = 1-cosine_similarity)\n",
    "distance_metric = losses.SiameseDistanceMetric.COSINE_DISTANCE\n",
    "\n",
    "#Negative pairs should have a distance of at least 0.3 (was 0.5 orig)\n",
    "margin = 0.5\n",
    "\n",
    "# ####  Configure the training #### \n",
    "# warmup_steps = math.ceil(len(train_dataset) * num_epochs / batch_size * 0.1) # 10% of train data for warm-up\n",
    "# print(\"Warmup-steps: {}\".format(warmup_steps))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "kJWvGNY7BJhY"
   },
   "outputs": [],
   "source": [
    "# ## results before finetuning\n",
    "# try:dev_eval(model) ## % with 7e-4 and min 4\n",
    "# except:()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 142
    },
    "id": "OuDei5zLgWFR",
    "outputId": "abbf863f-b5ef-4b42-b5f7-87e620b90f8b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ntrain_dataset = SentencesDataset(train_samples, model=model)\\n# DataLoader to batch your data\\ntrain_dataloader = DataLoader(train_dataset, batch_size= 128,#128,\\n                              shuffle=True)\\n## OnlineContrastiveLoss. - expects pairs\\n# train_loss = losses.OnlineContrastiveLoss(model=model, distance_metric=distance_metric, margin=margin)\\ntrain_loss = losses.CosineSimilarityLoss(model=model) #ORIG\\n\\n##ALT:\\ntrain_loss = losses.SoftmaxLoss(model=model,num_labels=1, sentence_embedding_dimension=384)\\n# Call the fit method\\ntry:\\n  model.fit(\\n      train_objectives=[(train_dataloader, train_loss)],\\n      epochs=1,\\n      # weight_decay=0,\\n      # scheduler='constantlr', \\n      optimizer_params={'lr': 1e-4 ## 3e-5\\n                        },\\n      show_progress_bar=True,\\n      use_amp=True,\\n      # evaluator=dev_eval, # ORIG - \\n      output_path='./cah_sbert_cos',\\n  )\\nexcept:()\\n\""
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "train_dataset = SentencesDataset(train_samples, model=model)\n",
    "# DataLoader to batch your data\n",
    "train_dataloader = DataLoader(train_dataset, batch_size= 128,#128,\n",
    "                              shuffle=True)\n",
    "## OnlineContrastiveLoss. - expects pairs\n",
    "# train_loss = losses.OnlineContrastiveLoss(model=model, distance_metric=distance_metric, margin=margin)\n",
    "train_loss = losses.CosineSimilarityLoss(model=model) #ORIG\n",
    "\n",
    "##ALT:\n",
    "train_loss = losses.SoftmaxLoss(model=model,num_labels=1, sentence_embedding_dimension=384)\n",
    "# Call the fit method\n",
    "try:\n",
    "  model.fit(\n",
    "      train_objectives=[(train_dataloader, train_loss)],\n",
    "      epochs=1,\n",
    "      # weight_decay=0,\n",
    "      # scheduler='constantlr', \n",
    "      optimizer_params={'lr': 1e-4 ## 3e-5\n",
    "                        },\n",
    "      show_progress_bar=True,\n",
    "      use_amp=True,\n",
    "      # evaluator=dev_eval, # ORIG - \n",
    "      output_path='./cah_sbert_cos',\n",
    "  )\n",
    "except:()\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 55
    },
    "id": "ufLRWUIWkZ-I",
    "outputId": "d85b744e-05bd-404e-fcea-4956080a03e3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n### dev eval\\n## example: https://jovian.ai/vumichien/sbert\\n# dev_eval(model) ## 0.555 Accracy (With min 6) , 57% with 1e-3 and min 3\\ntry:dev_eval(model) ## % with 7e-4 and min 4\\nexcept:()\\n'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "### dev eval\n",
    "## example: https://jovian.ai/vumichien/sbert\n",
    "# dev_eval(model) ## 0.555 Accracy (With min 6) , 57% with 1e-3 and min 3\n",
    "try:dev_eval(model) ## % with 7e-4 and min 4\n",
    "except:()\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84,
     "referenced_widgets": [
      "cfea03270fe64a44b54cde507f629eaf",
      "3f969476da554298b79fb91c4ec6e08f",
      "24c0db6a9c2146bea1b658c7245044aa",
      "82841c43fda44080b6debfd95e425b0f",
      "82cc1edd77f047b084a5f0c0b17a4bb1",
      "9952f2e5f62d402eadd85c0c17d5d8c2",
      "12f0eead030b48e79d989ab53ae75f5a",
      "ef2b365a6ece4bdb8d9cb84b6e593047",
      "35d4592d40f344ae92e27bd0bdc0f129",
      "d54fadb082c64e1e868ed9b21bdb4698",
      "9933a11516d64e00a16d8793fbb2ac04"
     ]
    },
    "id": "_CDwk0GxdAk0",
    "outputId": "3079a3bb-4ad0-4a70-e22a-67add369d1a4"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ce17816f2d7495a8fd8abbb023d978e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/7645 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 33min 47s\n",
      "Wall time: 28min 53s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "### could speed this up n^2 by getting embeddings only for unique combniations, then rejoining/merging...  (Won't work if using \"Text\" col)\n",
    "### ~ 1.5 minutes on colab GPU for 123K train samples , L6.  ~ 30-42 min with L12, full data\n",
    "### on 2060 local GPU, ~ 19 min for all train\n",
    "s1_emb = model.encode(list(df[USE_TEXT_COLS[0]].values),\n",
    "                      normalize_embeddings=False,\n",
    "                      show_progress_bar=True,\n",
    "                      batch_size=256,\n",
    "                    #   convert_to_tensor=True\n",
    "                    )\n",
    "if not ONE_COL_DATA_FORMAT:\n",
    "  ## black cards = quesiton/prompt\n",
    "  s2_emb = model.encode(list(df[USE_TEXT_COLS[1]].values),\n",
    "                        normalize_embeddings=False,\n",
    "                        show_progress_bar=True,\n",
    "                        batch_size=200,\n",
    "                      #   convert_to_tensor=True\n",
    "                      )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "04SnpNCFeDb4"
   },
   "outputs": [],
   "source": [
    "# try:\n",
    "#   dot_sim = []\n",
    "#   cos_score = []\n",
    "#   for i in range (len(s1_emb)):\n",
    "#       # dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "#       dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0])) ## am I getting the right cell with these two?? (cos and dot, [i][j] ? )\n",
    "#       cos_score.append(float(util.cos_sim(s1_emb[i], s2_emb[i])[0]))\n",
    "#   print(len(dot_sim))\n",
    "\n",
    "#   df[\"dot_sim_score\"] = dot_sim\n",
    "#   df[\"cos_sim_score\"] = cos_score\n",
    "#   print(df[[\"dot_sim_score\",\"cos_sim_score\",\"won\"]].corr().round(3))\n",
    "\n",
    "#   print(\"df:\\n\",df.groupby(\"won\")[[\"dot_sim_score\",\"cos_sim_score\"]].mean().round(3))\n",
    "\n",
    "#   # print(\"df_All:\")\n",
    "#   # df_all[\"dot_sim_score\"] = dot_sim\n",
    "#   # print(df_all[[\"dot_sim_score\",\"won\"]].corr())\n",
    "# except:()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4eT_LQFIiOhb"
   },
   "source": [
    "## Eval on df_Test\n",
    "* Can speed up by getting unique sentence pairs and their embeddings\n",
    "* Won't work the same for single text "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "Bv4dAPxkiXQK"
   },
   "outputs": [],
   "source": [
    "# df_test.drop_duplicates([\"black_card_text\",\"white_card_text\"]).shape[0] # 367K rows, vs 489 K for all "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84,
     "referenced_widgets": [
      "0b2804c196ef4f369aa469a3080cb930",
      "b17a4f8b6fca414286b47a65e60d1254",
      "6b8fcd36f1bb4a22981f0162ae538d7f",
      "62bb978e5a3b4a9cab47a8856c3633d0",
      "2941cf6f1bf745b7a8518ce2b1e8aaf7",
      "9f284c9e3c364f778c41b8c7e0ba3045",
      "030325446b8e4ede91020a4536f91064",
      "8c44dbe9d1c74aae894b1ff86ae69c9b",
      "75d0eb084e1b429b9375196c6069c55d",
      "b597b302d1d9405fbf9181d8a62e3944",
      "a5e9540e89ec4bee85fb45235c2fc640"
     ]
    },
    "id": "A6H_8-Ldirmw",
    "outputId": "6bada940-9caf-4a82-b16a-821d69afde77"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "314c690f9a54495d8e5d1f458489e36c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1912 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 8min 13s\n",
      "Wall time: 6min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "### could speed this up n^2 by getting embeddings only for unique combniations, then rejoining/merging...  (Won't work if using \"Text\" col)\n",
    "## change s1_emb to s1_emb_test\n",
    "## ~ 6 min for L12, local GPU, full test\n",
    "s1_emb_test = model.encode(list(df_test[USE_TEXT_COLS[0]].values),\n",
    "                      normalize_embeddings=False,\n",
    "                      show_progress_bar=True,\n",
    "                      batch_size=256,\n",
    "                    #   convert_to_tensor=True\n",
    "                    )\n",
    "if not ONE_COL_DATA_FORMAT:\n",
    "  ## black cards = quesiton/prompt\n",
    "  s2_emb_test = model.encode(list(df_test[USE_TEXT_COLS[1]].values),\n",
    "                        normalize_embeddings=False,\n",
    "                        show_progress_bar=True,\n",
    "                        batch_size=256,\n",
    "                      #   convert_to_tensor=True\n",
    "                      )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QWBwiv7mirq-",
    "outputId": "ec516967-cfca-41c1-8dc0-7ce8afc82ace"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if not ONE_COL_DATA_FORMAT:\n",
    "  try:\n",
    "    dot_sim = []\n",
    "    cos_score = []\n",
    "    for i in range (len(s1_emb)):\n",
    "        # dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "        dot_sim.append(float(util.dot_score(s1_emb[i], s2_emb[i])[0])) ## am I getting the right cell with these two?? (cos and dot, [i][j] ? )\n",
    "        cos_score.append(float(util.cos_sim(s1_emb[i], s2_emb[i])[0]))\n",
    "    print(len(dot_sim))\n",
    "\n",
    "    df_test[\"dot_sim_score\"] = dot_sim\n",
    "    df_test[\"cos_sim_score\"] = cos_score\n",
    "    print(df_test[[\"dot_sim_score\",\"cos_sim_score\",\"won\"]].corr().round(3))\n",
    "\n",
    "    print(\"df_test:\\n\",df_test.groupby(\"won\")[[\"dot_sim_score\",\"cos_sim_score\"]].mean().round(3))\n",
    "\n",
    "    print(\"Acc:\",df_test.sort_values(\"dot_sim_score\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 17.5% acc\n",
    "    print(\"Acc: (dot)\",df_test.sort_values(\"dot_sim_score\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean())\n",
    "    print(\"Acc:\",df_test.sort_values(\"cos_sim_score\",ascending=True).groupby(\"fake_round_id\").head(1)[\"won\"].mean()) ## 0.05\n",
    "    print(\"Acc:\",df_test.sort_values(\"cos_sim_score\",ascending=False).groupby(\"fake_round_id\").head(1)[\"won\"].mean())\n",
    "  except:()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rkIPD7_uXw6D"
   },
   "source": [
    "### Pretrain - unsupervised\n",
    "* TSDAE or other method ? \n",
    "* https://www.sbert.net/examples/unsupervised_learning/TSDAE/README.html#tsdae-as-pre-training-task\n",
    "\n",
    "\n",
    "ST recommends MultipleNegativesRankingLoss\n",
    "* https://www.sbert.net/examples/training/nli/README.html#multiplenegativesrankingloss\n",
    "*  MultipleNegativesRankingLoss only requires positive pairs, i.e., we only need examples of positive/funny pairs. (BUT It also supports hard negatives in a triplet)\n",
    "\n",
    "Also: OnlineContrastiveLoss\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html#training\n",
    "* Constrative Loss / `losses.OnlineContrastiveLoss`\n",
    "* Choosing the distance function and especially choosing a sensible margin are quite important for the success of constrative loss. In the given example, we use cosine_distance (which is 1-cosine_similarity) with a margin of 0.5. I.e., non-duplicate questions should have a cosine_distance of at least 0.5 (which is equivalent to a 0.5 cosine similarity difference).\n",
    "* An improved version of constrative loss is OnlineConstrativeLoss, which looks which negative pairs have a lower distance that the largest positive pair and which positive pairs have a higher distance than the lowest distance of negative pairs. I.e., this loss automatically detects the hard cases in a batch and computes the loss only for these cases.\n",
    "\n",
    "Can also do BOTH losses, as in:\n",
    "\n",
    "* Multi-Task-Learning\n",
    "    Constrative Loss works well for pair classification, i.e., given two pairs, are these duplicates or not. It pushes negative pairs far away in vector space, so that the distinguishing between duplicate and non-duplicate pairs works good.\n",
    "\n",
    "    MultipleNegativesRankingLoss on the other sides mainly reduces the distance between positive pairs out of large set of possible candidates. However, the distance between non-duplicate questions is not so large, so that this loss does not work that weill for pair classification.\n",
    "\n",
    "* https://www.sbert.net/examples/training/quora_duplicate_questions/README.html#multi-task-learning\n",
    "\n",
    "More losses (e.g. triplet): https://www.sbert.net/docs/package_reference/losses.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "CAlPTP7_XwVp"
   },
   "outputs": [],
   "source": [
    "# # Define your sentence transformer model using CLS pooling\n",
    "# model_name = 'bert-base-uncased'\n",
    "# word_embedding_model = models.Transformer(model_name)\n",
    "# pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(), 'cls')\n",
    "# model = SentenceTransformer(modules=[word_embedding_model, pooling_model])\n",
    "\n",
    "# # Define a list with sentences (1k - 100k sentences)\n",
    "# train_sentences = [\"Your set of sentences\",\n",
    "#                    \"Model will automatically add the noise\", \n",
    "#                    \"And re-construct it\",\n",
    "#                    \"You should provide at least 1k sentences\"]\n",
    "\n",
    "# # Create the special denoising dataset that adds noise on-the-fly\n",
    "# train_dataset = datasets.DenoisingAutoEncoderDataset(train_sentences)\n",
    "\n",
    "# # DataLoader to batch your data\n",
    "# train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "\n",
    "# # Use the denoising auto-encoder loss\n",
    "# train_loss = losses.DenoisingAutoEncoderLoss(model, decoder_name_or_path=model_name, tie_encoder_decoder=True)\n",
    "\n",
    "# # Call the fit method\n",
    "# model.fit(\n",
    "#     train_objectives=[(train_dataloader, train_loss)],\n",
    "#     epochs=1,\n",
    "#     weight_decay=0,\n",
    "#     scheduler='constantlr',\n",
    "#     optimizer_params={'lr': 3e-5},\n",
    "#     show_progress_bar=True\n",
    "# )\n",
    "\n",
    "# model.save('output/tsdae-model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8UwMrw__mpbH"
   },
   "source": [
    "#### get embeddings & dot product/codine distance over all \n",
    "\n",
    "* `util.semantic_search` - could do this is one step for us (but we'll want to train a model on embeddings anyway) \n",
    "\n",
    "* https://www.sbert.net/docs/usage/semantic_textual_similarity.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "h-4YyJH1mwOJ"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ## white cards = most predictive information\n",
    "\n",
    "# # ### encode_multi_process ? (needs pool enabled)\n",
    "# ##     pool = model.start_multi_process_pool(encode_batch_size=2000)\n",
    "# ##     embeddings = model.encode_multi_process(texts, pool)\n",
    "\n",
    "# s1_emb = model.encode(list(df[\"white_card_text\"].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=128,\n",
    "#                     #   convert_to_tensor=True \n",
    "#                       )#.astype(np.float32) ## 40s for 4K , with L12 miniLM, on cpu\n",
    "# ### 7 min for all min7 (37k) with L12\n",
    "# print(s1_emb.shape,\"s1_emb.shape\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "c84GZCIOm6rc"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### 8.7 min on colab for L12, min 4 (371K)\n",
    "\n",
    "# s1_emb = model.encode(list(df[\"white_card_text\"].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=128,\n",
    "#                       normalize_embeddings=True\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )\n",
    "\n",
    "# ## black cards = quesiton/prompt\n",
    "# s2_emb = model.encode(list(df[\"black_card_text\"].values),\n",
    "#                       normalize_embeddings=True,\n",
    "#                       show_progress_bar=True,\n",
    "#                       batch_size=256,\n",
    "#                       normalize_embeddings=True\n",
    "#                     #   convert_to_tensor=True\n",
    "#                     )#.astype(np.float32) ## 1.5 min for 4K , with L12 miniLM, on cpu\n",
    "# print(s2_emb.shape)\n",
    "\n",
    "\n",
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "# print(len(dot_sim))\n",
    "# dot_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "-HS3rT63uW6J"
   },
   "outputs": [],
   "source": [
    "# len(s2_emb[0]) # 384 = dim of the \"first\" sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "LUejcarOuocO"
   },
   "outputs": [],
   "source": [
    "# ### Hopefully I am doing this correctly ??? \n",
    "# ### get cosine similarity \n",
    "# output = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     output.append(util.cos_sim(s1_emb[i], s2_emb[i])[0])\n",
    "# print(len(output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZQYpl70Cn9aP",
    "outputId": "2426f254-606d-424b-ba33-b2441a3a81ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Compute cosine similarity between all pairs - outputs matrix of shape S1 X S2 (i.e # samples = ineffecient in memory!)\n",
    "# output = util.pytorch_cos_sim(s1_emb, s2_emb)\n",
    "# output = util.cos_sim(s1_emb, s2_emb) # ORIG, nXn matrix\n",
    "# print(output.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "g8plFSPvsN2C"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### Hopefully I am doing this correctly ??? \n",
    "# # # ## dot product:\n",
    "# # dot_sim = util.dot_score(s1_emb, s2_emb) # memory crash? \n",
    "# # print(dot_sim.shape)\n",
    "\n",
    "# dot_sim = []\n",
    "# for i in range (len(s1_emb)):\n",
    "#     dot_sim.append(util.dot_score(s1_emb[i], s2_emb[i])[0])\n",
    "# print(len(dot_sim))\n",
    "# dot_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "4dO5Fro2AEJY"
   },
   "outputs": [],
   "source": [
    "# print(\"cos done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7ewqEcHfqgGW",
    "outputId": "618ae015-3984-4c2f-d169-8fe373bad667"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# from torch.nn import CosineSimilarity, PairwiseDistance\n",
    "# ## https://pytorch.org/docs/stable/generated/torch.nn.CosineSimilarity.html\n",
    "# nn_cos = CosineSimilarity(dim=1, eps=1e-6)\n",
    "# output = nn_cos(s1_emb, s2_emb) \n",
    "# print(\"output\",output.shape)\n",
    "# nn_pairwiseDist = PairwiseDistance()\n",
    "# output_2 = nn_pairwiseDist(s1_emb, s2_emb)\n",
    "# print(\"output_2\",output_2.shape)\n",
    "\n",
    "# # ## dot product:\n",
    "# output_3 = util.dot_score(s1_emb, s2_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "5bpVLWPqqgM4"
   },
   "outputs": [],
   "source": [
    "# df[\"cos_sim\"] = output\n",
    "# df[\"pairwiseDist_sim\"] = output_2\n",
    "# df[\"dot_score\"] = output_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "_pV02TX0q9H8"
   },
   "outputs": [],
   "source": [
    "# df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "4mJQsyAPrIPh"
   },
   "outputs": [],
   "source": [
    "# df.groupby(\"picks\")[[\"cos_sim\",\"pairwiseDist_sim\"]].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Uj7QLQmVt-7S"
   },
   "source": [
    "#### Model on embeddings \n",
    "* linear model on embeddings per sentence and cossim, pairwise dist score;\n",
    "* + difference, +- multiplication of vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "6kZfxLXWEF93"
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ### mean diff, math/ mult per row: \n",
    "\n",
    "# ## https://stackoverflow.com/questions/50430585/mean-difference-of-two-numpy-arrays\n",
    "# # np.mean(np.abs(s1_emb[:, None] - s2_emb))\n",
    "# vector_diffs = s1_emb - s2_emb\n",
    "# mean_diff = np.mean(vector_diffs,axis=1) # 1 col\n",
    "# max_diff = np.min(vector_diffs,axis=1) \n",
    "# min_diff = np.max(vector_diffs,axis=1) \n",
    "# # np_dot = np.dot(s1_emb,s2_emb)\n",
    "# print(mean_diff)\n",
    "# print(max_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 641
    },
    "id": "_T3ooMlWiVsT",
    "outputId": "d89f8a79-64c2-4aa1-9bc4-11a64a474150"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fake_round_id</th>\n",
       "      <th>black_card_text</th>\n",
       "      <th>white_card_text</th>\n",
       "      <th>ID_index</th>\n",
       "      <th>id_white</th>\n",
       "      <th>round_completion_seconds</th>\n",
       "      <th>won</th>\n",
       "      <th>text</th>\n",
       "      <th>sum_won</th>\n",
       "      <th>pair_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>141510</td>\n",
       "      <td>__. Awesome in theory, kind of a mess in pract...</td>\n",
       "      <td>The eight gay warlocks who dictate the rules o...</td>\n",
       "      <td>2</td>\n",
       "      <td>1810</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>The eight gay warlocks who dictate the rules o...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>29064</td>\n",
       "      <td>I'll take the BBQ bacon burger with a fried eg...</td>\n",
       "      <td>Frantically writing equations on a chalkboard</td>\n",
       "      <td>7</td>\n",
       "      <td>813</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>I'll take the BBQ bacon burger with a fried eg...</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>204306</td>\n",
       "      <td>I've got rhythm, I've got music, I've got __. ...</td>\n",
       "      <td>The fear and hatred in men's hearts</td>\n",
       "      <td>4</td>\n",
       "      <td>1821</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>I've got rhythm, I've got music, I've got The ...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>144997</td>\n",
       "      <td>What never fails to liven up the party?</td>\n",
       "      <td>The best taquito in the galaxy</td>\n",
       "      <td>3</td>\n",
       "      <td>1781</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>What never fails to liven up the party? The be...</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>199</td>\n",
       "      <td>And today's soup is Cream of __.</td>\n",
       "      <td>Taking a shit while running at full speed</td>\n",
       "      <td>5</td>\n",
       "      <td>1700</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>And today's soup is Cream of Taking a shit whi...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957075</th>\n",
       "      <td>59190</td>\n",
       "      <td>Just saw this upsetting video! Please retweet!...</td>\n",
       "      <td>Fake tits</td>\n",
       "      <td>9</td>\n",
       "      <td>755</td>\n",
       "      <td>63</td>\n",
       "      <td>0</td>\n",
       "      <td>Just saw this upsetting video! Please retweet!...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957076</th>\n",
       "      <td>55070</td>\n",
       "      <td>Oh my god! __ killed Kenny!</td>\n",
       "      <td>Being a total Miranda</td>\n",
       "      <td>7</td>\n",
       "      <td>420</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>Oh my god! Being a total Miranda killed Kenny!</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957077</th>\n",
       "      <td>114606</td>\n",
       "      <td>Madam President, the asteroid is headed direct...</td>\n",
       "      <td>Telling Heather she can't pull off that top</td>\n",
       "      <td>1</td>\n",
       "      <td>1711</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>Madam President, the asteroid is headed direct...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957078</th>\n",
       "      <td>110038</td>\n",
       "      <td>When I pooped, what came out of my butt?</td>\n",
       "      <td>Eating a pizza that's lying in the street to g...</td>\n",
       "      <td>1</td>\n",
       "      <td>703</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>When I pooped, what came out of my butt? Eatin...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1957079</th>\n",
       "      <td>26601</td>\n",
       "      <td>Why am I laughing and crying and taking off my...</td>\n",
       "      <td>Letting this loser eat me out</td>\n",
       "      <td>8</td>\n",
       "      <td>1134</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>Why am I laughing and crying and taking off my...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1957080 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         fake_round_id                                    black_card_text  \\\n",
       "0               141510  __. Awesome in theory, kind of a mess in pract...   \n",
       "1                29064  I'll take the BBQ bacon burger with a fried eg...   \n",
       "2               204306  I've got rhythm, I've got music, I've got __. ...   \n",
       "3               144997            What never fails to liven up the party?   \n",
       "4                  199                   And today's soup is Cream of __.   \n",
       "...                ...                                                ...   \n",
       "1957075          59190  Just saw this upsetting video! Please retweet!...   \n",
       "1957076          55070                        Oh my god! __ killed Kenny!   \n",
       "1957077         114606  Madam President, the asteroid is headed direct...   \n",
       "1957078         110038           When I pooped, what came out of my butt?   \n",
       "1957079          26601  Why am I laughing and crying and taking off my...   \n",
       "\n",
       "                                           white_card_text  ID_index  \\\n",
       "0        The eight gay warlocks who dictate the rules o...         2   \n",
       "1            Frantically writing equations on a chalkboard         7   \n",
       "2                      The fear and hatred in men's hearts         4   \n",
       "3                           The best taquito in the galaxy         3   \n",
       "4                Taking a shit while running at full speed         5   \n",
       "...                                                    ...       ...   \n",
       "1957075                                          Fake tits         9   \n",
       "1957076                              Being a total Miranda         7   \n",
       "1957077        Telling Heather she can't pull off that top         1   \n",
       "1957078  Eating a pizza that's lying in the street to g...         1   \n",
       "1957079                      Letting this loser eat me out         8   \n",
       "\n",
       "         id_white  round_completion_seconds  won  \\\n",
       "0            1810                        45    0   \n",
       "1             813                        22    0   \n",
       "2            1821                         5    0   \n",
       "3            1781                        15    0   \n",
       "4            1700                        41    0   \n",
       "...           ...                       ...  ...   \n",
       "1957075       755                        63    0   \n",
       "1957076       420                        29    0   \n",
       "1957077      1711                        29    0   \n",
       "1957078       703                        19    0   \n",
       "1957079      1134                         9    0   \n",
       "\n",
       "                                                      text  sum_won  \\\n",
       "0        The eight gay warlocks who dictate the rules o...        0   \n",
       "1        I'll take the BBQ bacon burger with a fried eg...        1   \n",
       "2        I've got rhythm, I've got music, I've got The ...        0   \n",
       "3        What never fails to liven up the party? The be...        0   \n",
       "4        And today's soup is Cream of Taking a shit whi...        0   \n",
       "...                                                    ...      ...   \n",
       "1957075  Just saw this upsetting video! Please retweet!...        1   \n",
       "1957076     Oh my god! Being a total Miranda killed Kenny!        0   \n",
       "1957077  Madam President, the asteroid is headed direct...        0   \n",
       "1957078  When I pooped, what came out of my butt? Eatin...        0   \n",
       "1957079  Why am I laughing and crying and taking off my...        0   \n",
       "\n",
       "         pair_count  \n",
       "0                 3  \n",
       "1                 5  \n",
       "2                 2  \n",
       "3                 5  \n",
       "4                 4  \n",
       "...             ...  \n",
       "1957075           3  \n",
       "1957076           4  \n",
       "1957077           3  \n",
       "1957078           2  \n",
       "1957079           1  \n",
       "\n",
       "[1957080 rows x 10 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_HurL_DrAK68",
    "outputId": "0c37e546-ad45-45f4-b36d-e494050e4141"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1957080, 768)\n",
      "(489280, 768)\n"
     ]
    }
   ],
   "source": [
    "y = df[\"won\"].values\n",
    "# X = np.concatenate([s1_emb,s2_emb],axis=1) # BOTH \n",
    "\n",
    "X =s1_emb\n",
    "X_test = s1_emb_test\n",
    "# try:\n",
    "#   X = s2_emb # BOTH \n",
    "#   X_test = s2_emb_test\n",
    "# except:\n",
    "#   X =s1_emb\n",
    "#   X_test = s1_emb_test\n",
    "# X = np.concatenate([s1_emb,vector_diffs],axis=1)## ALT - white cards + diffs\n",
    "\n",
    "print(X.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "AWxHrgvdt9kj"
   },
   "outputs": [],
   "source": [
    "# X = np.column_stack((X,output)) ### memory crash ? \n",
    "# X = np.column_stack((X,dot_sim)) ## dot product \n",
    "\n",
    "# # X = np.column_stack((X,output_2))\n",
    "# # X = np.column_stack((X,output_3))\n",
    "# X = np.column_stack((X,mean_diff))\n",
    "# X = np.column_stack((X,max_diff))\n",
    "# X = np.column_stack((X,min_diff))\n",
    "# print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "id": "NZWgLq44wVT7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "\n",
      "roc_auc 59.040000000000006\n",
      "proba based:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96    440352\n",
      "           1       1.00      0.17      0.29     48928\n",
      "\n",
      "    accuracy                           0.92    489280\n",
      "   macro avg       0.96      0.58      0.62    489280\n",
      "weighted avg       0.92      0.92      0.89    489280\n",
      "\n",
      "rocAUC: 0.5904\n",
      "top 1 acc by round: 16.90\n",
      "top 2 acc by round: 15.34\n",
      "top 3 acc by round: 14.22\n",
      "top 1 acc by round: 0.169\n",
      "top 2 acc by round: 0.3068\n",
      "top 3 acc by round: 0.4266\n",
      "CPU times: total: 10min 11s\n",
      "Wall time: 11min 40s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# StandardScaler\n",
    "clf = make_pipeline(StandardScaler(),  LogisticRegression(solver=\"sag\")) ## PCA(n_components=2), # error with array? \n",
    "# clf = make_pipeline(StandardScaler(),  LogisticRegression())\n",
    "# clf = LogisticRegression() ## same res. ~ 3 min\n",
    "# cv_preds = cross_val_predict(clf,X,y,n_jobs=-2)\n",
    "\n",
    "clf.fit(X,y)\n",
    "y_test_preds = clf.predict(X_test)\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds))\n",
    "y_test_preds_proba = clf.predict_proba(X_test)[:,1]\n",
    "print(\"\\nroc_auc\",100*round(roc_auc_score(y_true=y_test,y_score=y_test_preds_proba),4))\n",
    "print(\"proba based:\")\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_proba>=0.5))\n",
    "\n",
    "eval_preds(df_test,y_test_preds_proba)\n",
    "### on sample - roc_auc 59.25 \n",
    "# \"\"\"\n",
    "# cv_preds = cross_val_predict(clf,X,y,method=\"predict_proba\",cv=3,n_jobs=-2)[:,1]\n",
    "# print(classification_report(y_true=y,y_pred=cv_preds>=0.5))\n",
    "# print(\"roc_auc\",roc_auc_score(y_true=y,y_score=cv_preds))\n",
    "\n",
    "\n",
    "# # #### min 4 (joint text), v1, 371K records\n",
    "# #               precision    recall  f1-score   support\n",
    "\n",
    "# #            0       0.65      0.90      0.75    227115\n",
    "# #            1       0.58      0.22      0.32    144299\n",
    "\n",
    "# #     accuracy                           0.64    371414\n",
    "# #    macro avg       0.61      0.56      0.54    371414\n",
    "# # weighted avg       0.62      0.64      0.58    371414\n",
    "\n",
    "# # roc_auc 0.620\n",
    "\n",
    "\n",
    "# \"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* RES:\n",
    "\n",
    "\n",
    "              precision    recall  f1-score   support\n",
    "\n",
    "           0       0.90      1.00      0.95    440352\n",
    "           1       0.00      0.00      0.00     48928\n",
    "\n",
    "    accuracy                           0.90    489280\n",
    "   macro avg       0.45      0.50      0.47    489280\n",
    "weighted avg       0.81      0.90      0.85    489280\n",
    "\n",
    "\n",
    "roc_auc 59.040000000000006\n",
    "\n",
    "proba based:\n",
    "\n",
    "              precision    recall  f1-score   support\n",
    "\n",
    "           0       0.90      1.00      0.95    440352\n",
    "           1       0.00      0.00      0.00     48928\n",
    "\n",
    "    accuracy                           0.90    489280\n",
    "   macro avg       0.45      0.50      0.47    489280\n",
    "weighted avg       0.81      0.90      0.85    489280\n",
    "\n",
    "              precision    recall  f1-score   support\n",
    "\n",
    "           0       0.92      1.00      0.96    440352\n",
    "           1       1.00      0.17      0.29     48928\n",
    "\n",
    "    accuracy                           0.92    489280\n",
    "   macro avg       0.96      0.58      0.62    489280\n",
    "weighted avg       0.92      0.92      0.89    489280\n",
    "\n",
    "rocAUC: 0.5904\n",
    "top 1 acc by round: 16.90\n",
    "top 2 acc by round: 15.34\n",
    "top 3 acc by round: 14.22\n",
    "top 1 acc by round: 0.169\n",
    "top 2 acc by round: 0.3068\n",
    "top 3 acc by round: 0.4266"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "C571NK8Ozcqz"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "\n",
      "roc_auc 50.0\n",
      "proba based:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\apps\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95    440352\n",
      "           1       0.00      0.00      0.00     48928\n",
      "\n",
      "    accuracy                           0.90    489280\n",
      "   macro avg       0.45      0.50      0.47    489280\n",
      "weighted avg       0.81      0.90      0.85    489280\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    440352\n",
      "           1       1.00      1.00      1.00     48928\n",
      "\n",
      "    accuracy                           1.00    489280\n",
      "   macro avg       1.00      1.00      1.00    489280\n",
      "weighted avg       1.00      1.00      1.00    489280\n",
      "\n",
      "rocAUC: 0.5\n",
      "top 1 acc by round: 10.03\n",
      "top 2 acc by round: 9.98\n",
      "top 3 acc by round: 9.94\n",
      "top 1 acc by round: 0.1003\n",
      "top 2 acc by round: 0.1995\n",
      "top 3 acc by round: 0.2983\n",
      "CPU times: total: 9h 14min 58s\n",
      "Wall time: 1h 25min 44s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## slow... ?  30 min\n",
    "clf_rf = RandomForestClassifier(min_samples_split=5, min_samples_leaf=12,#,max_depth=9,\n",
    "                                n_jobs=-2,ccp_alpha=0.005) # n_jobs=-2\n",
    "\n",
    "# cv_preds_2 = cross_val_predict(clf_rf,X,y,method=\"predict_proba\",cv=3)[:,1]\n",
    "# print(classification_report(y_true=y,y_pred=cv_preds_2>=0.5))\n",
    "# print(\"roc_auc\",roc_auc_score(y_true=y,y_score=cv_preds_2))\n",
    "\n",
    "clf_rf.fit(X,y)\n",
    "print(\"Trained\")\n",
    "y_test_preds_rf = clf_rf.predict(X_test)\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_rf))\n",
    "y_test_preds_proba_rf = clf_rf.predict_proba(X_test)[:,1]\n",
    "print(\"\\nroc_auc\",100*round(roc_auc_score(y_true=y_test,y_score=y_test_preds_proba_rf),4))\n",
    "print(\"proba based:\")\n",
    "print(classification_report(y_true=y_test,y_pred=y_test_preds_proba_rf>=0.5))\n",
    "\n",
    "eval_preds(df_test,y_test_preds_proba_rf)\n",
    "# # #### min 4 (joint text), v1, 371K records\n",
    "#               precision    recall  f1-score   support\n",
    "\n",
    "#            0       0.63      0.94      0.76    227115\n",
    "#            1       0.61      0.14      0.23    144299\n",
    "\n",
    "#     accuracy                           0.63    371414\n",
    "#    macro avg       0.62      0.54      0.49    371414\n",
    "# weighted avg       0.62      0.63      0.55    371414\n",
    "\n",
    "# roc_auc 0.609\n",
    "\n",
    "\n",
    "\n",
    "##TODO - feature importance (do the deltas improve model?)\n",
    "\n",
    "# min 2 , L12 miniLM\n",
    "#               precision    recall  f1-score   support\n",
    "\n",
    "#            0       0.73      0.97      0.83    469455\n",
    "#            1       0.53      0.10      0.17    183561\n",
    "\n",
    "#     accuracy                           0.72    653016\n",
    "#    macro avg       0.63      0.53      0.50    653016\n",
    "# weighted avg       0.68      0.72      0.65    653016\n",
    "\n",
    "# roc_auc 0.64871\n",
    "# CPU times: user 2h "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "G478iE9fuHSW"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    440352\n",
      "           1       1.00      1.00      1.00     48928\n",
      "\n",
      "    accuracy                           1.00    489280\n",
      "   macro avg       1.00      1.00      1.00    489280\n",
      "weighted avg       1.00      1.00      1.00    489280\n",
      "\n",
      "rocAUC: 0.5\n",
      "top 1 acc by round: 10.09\n",
      "top 2 acc by round: 10.00\n",
      "top 3 acc by round: 9.97\n",
      "top 1 acc by round: 0.1009\n",
      "top 2 acc by round: 0.2001\n",
      "top 3 acc by round: 0.2992\n"
     ]
    }
   ],
   "source": [
    "eval_preds(df_test,y_test_preds_proba_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RES: (Random forest):\n",
    "              precision    recall  f1-score   support\n",
    "\n",
    "           0       1.00      1.00      1.00    440352\n",
    "           1       1.00      1.00      1.00     48928\n",
    "\n",
    "    accuracy                           1.00    489280\n",
    "   macro avg       1.00      1.00      1.00    489280\n",
    "weighted avg       1.00      1.00      1.00    489280\n",
    "\n",
    "rocAUC: 0.5\n",
    "top 1 acc by round: 10.09\n",
    "top 2 acc by round: 10.00\n",
    "top 3 acc by round: 9.97\n",
    "top 1 acc by round: 0.1009\n",
    "top 2 acc by round: 0.2001\n",
    "top 3 acc by round: 0.2992"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "WIEDabG88ax8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"text\"].str.split().str.len().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "id": "RVhmVMGq3Vvi"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n##ORIG:\\nX_train, X_test, y_train, y_test = train_test_split(\\n    list(df[\"text\"].values), list(df[\"won\"].values), test_size=0.25, random_state=42)\\n'"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "# y = df[\"picks\"].values\n",
    "# # X = np.concatenate([s1_emb,s2_emb],axis=1) # BOTH \n",
    "# X = list(df[\"black_card_text\"].values)\n",
    "\"\"\"\n",
    "##ORIG:\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    list(df[\"text\"].values), list(df[\"won\"].values), test_size=0.25, random_state=42)\n",
    "\"\"\"\n",
    "## NEW:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "1WEZ5crL80SI"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0, 1}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "id": "qPs1eIc14Ei0"
   },
   "outputs": [],
   "source": [
    "# Try finetuning? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "pdKgjj3tzcx4"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50ccb61168ad40cda1b45e49197cd988",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a296057d135543a5be7f23cb6141f494",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/420M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ca7c53942764e23aceae3053d3f31c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67d87af72dd245548c5f16a7ea73719f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2dafe9ec6c64d5dab2a5fcb26f10759",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "### https://www.sbert.net/docs/training/overview.html\n",
    "from torch import nn\n",
    "\n",
    "word_embedding_model = models.Transformer('bert-base-uncased', max_seq_length=256)\n",
    "pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())\n",
    "dense_model = models.Dense(in_features=pooling_model.get_sentence_embedding_dimension(),\n",
    "                           out_features=64, # 128\n",
    "                           activation_function=nn.Tanh())\n",
    "\n",
    "# model2 = SentenceTransformer(modules=[word_embedding_model, pooling_model, dense_model])\n",
    "model2 = SentenceTransformer(modules=[model, dense_model])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "id": "_5gL5gVG8rAt"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1957080"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "id": "9lZuN3eO5Ex7"
   },
   "outputs": [],
   "source": [
    "# [X_train[0:5],y_train[0:5]]\n",
    "## list(df[\"black_card_text\"].values)\n",
    "train_examples =  [InputExample(texts=[X_train[i]],label=float(y_train[i])) for i in range(len(X_train)-1)]\n",
    "# [X_train[0:5],y_train[0:5]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "id": "jC_4K4HK7vDg"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The eight gay warlocks who dictate the rules of fashion. Awesome in theory, kind of a mess in practice.'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "id": "eAQC1-uI7sA3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1957080"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "id": "2o8dxHLV7osN"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1957080"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "id": "JKBO4Rj67lsZ"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1957079"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "id": "smasDpEz38-E"
   },
   "outputs": [],
   "source": [
    "### IndexError: list index out of range. ? \n",
    "\n",
    "\n",
    "#Define your train examples. You need more than just two examples...\n",
    "# train_examples = [InputExample(texts=['My first sentence', 'My second sentence'], label=0.8),\n",
    "#     InputExample(texts=['Another pair', 'Unrelated sentence'], label=0.3)]\n",
    "\n",
    "# train_examples =  [X_train[0:5],y_train[0:5]]\n",
    "\n",
    "#Define your train dataset, the dataloader and the train loss\n",
    "train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=128)\n",
    "# train_loss = losses.CosineSimilarityLoss(model)\n",
    "\n",
    "# #Tune the model\n",
    "# model2.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "id": "izHedCfQ985E"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\apps\\anaconda3\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20165624ceb94e88908c0ce0d6ff6d63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57cb1a5c62414e3c9d7a0fc129018ce4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/7645 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 36.00 MiB (GPU 0; 6.00 GiB total capacity; 5.14 GiB already allocated; 0 bytes free; 5.30 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Input \u001b[1;32mIn [66]\u001b[0m, in \u001b[0;36m<cell line: 8>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      5\u001b[0m train_loss \u001b[38;5;241m=\u001b[39m losses\u001b[38;5;241m.\u001b[39mBatchSemiHardTripletLoss(model\u001b[38;5;241m=\u001b[39mmodel2)\n\u001b[0;32m      7\u001b[0m \u001b[38;5;66;03m#Tune the model\u001b[39;00m\n\u001b[1;32m----> 8\u001b[0m \u001b[43mmodel2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_objectives\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_loss\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwarmup_steps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m50\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\sentence_transformers\\SentenceTransformer.py:712\u001b[0m, in \u001b[0;36mSentenceTransformer.fit\u001b[1;34m(self, train_objectives, evaluator, epochs, steps_per_epoch, scheduler, warmup_steps, optimizer_class, optimizer_params, weight_decay, evaluation_steps, output_path, save_best_model, max_grad_norm, use_amp, callback, show_progress_bar, checkpoint_path, checkpoint_save_steps, checkpoint_save_total_limit)\u001b[0m\n\u001b[0;32m    710\u001b[0m     skip_scheduler \u001b[38;5;241m=\u001b[39m scaler\u001b[38;5;241m.\u001b[39mget_scale() \u001b[38;5;241m!=\u001b[39m scale_before_step\n\u001b[0;32m    711\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 712\u001b[0m     loss_value \u001b[38;5;241m=\u001b[39m \u001b[43mloss_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    713\u001b[0m     loss_value\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[0;32m    714\u001b[0m     torch\u001b[38;5;241m.\u001b[39mnn\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39mclip_grad_norm_(loss_model\u001b[38;5;241m.\u001b[39mparameters(), max_grad_norm)\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\sentence_transformers\\losses\\BatchSemiHardTripletLoss.py:43\u001b[0m, in \u001b[0;36mBatchSemiHardTripletLoss.forward\u001b[1;34m(self, sentence_features, labels)\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, sentence_features: Iterable[Dict[\u001b[38;5;28mstr\u001b[39m, Tensor]], labels: Tensor):\n\u001b[1;32m---> 43\u001b[0m     rep \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msentence_embedder\u001b[49m\u001b[43m(\u001b[49m\u001b[43msentence_features\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msentence_embedding\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m     44\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_semi_hard_triplet_loss(labels, rep)\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\container.py:141\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    139\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    140\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 141\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m    142\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\container.py:141\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    139\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    140\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 141\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m    142\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\sentence_transformers\\models\\Transformer.py:66\u001b[0m, in \u001b[0;36mTransformer.forward\u001b[1;34m(self, features)\u001b[0m\n\u001b[0;32m     63\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtoken_type_ids\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m features:\n\u001b[0;32m     64\u001b[0m     trans_features[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtoken_type_ids\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m features[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtoken_type_ids\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m---> 66\u001b[0m output_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mauto_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mtrans_features\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m     67\u001b[0m output_tokens \u001b[38;5;241m=\u001b[39m output_states[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m     69\u001b[0m features\u001b[38;5;241m.\u001b[39mupdate({\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtoken_embeddings\u001b[39m\u001b[38;5;124m'\u001b[39m: output_tokens, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mattention_mask\u001b[39m\u001b[38;5;124m'\u001b[39m: features[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mattention_mask\u001b[39m\u001b[38;5;124m'\u001b[39m]})\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\transformers\\models\\albert\\modeling_albert.py:737\u001b[0m, in \u001b[0;36mAlbertModel.forward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m    732\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[0;32m    734\u001b[0m embedding_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membeddings(\n\u001b[0;32m    735\u001b[0m     input_ids, position_ids\u001b[38;5;241m=\u001b[39mposition_ids, token_type_ids\u001b[38;5;241m=\u001b[39mtoken_type_ids, inputs_embeds\u001b[38;5;241m=\u001b[39minputs_embeds\n\u001b[0;32m    736\u001b[0m )\n\u001b[1;32m--> 737\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    738\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    739\u001b[0m \u001b[43m    \u001b[49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    740\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    741\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    742\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    743\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    744\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    746\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    748\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler_activation(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output[:, \u001b[38;5;241m0\u001b[39m])) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\transformers\\models\\albert\\modeling_albert.py:484\u001b[0m, in \u001b[0;36mAlbertTransformer.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m    481\u001b[0m \u001b[38;5;66;03m# Index of the hidden group\u001b[39;00m\n\u001b[0;32m    482\u001b[0m group_idx \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(i \u001b[38;5;241m/\u001b[39m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_groups))\n\u001b[1;32m--> 484\u001b[0m layer_group_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43malbert_layer_groups\u001b[49m\u001b[43m[\u001b[49m\u001b[43mgroup_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    485\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    486\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    487\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m[\u001b[49m\u001b[43mgroup_idx\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mlayers_per_group\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mgroup_idx\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mlayers_per_group\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    488\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    489\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    490\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    491\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_group_output[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    493\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m output_attentions:\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\transformers\\models\\albert\\modeling_albert.py:436\u001b[0m, in \u001b[0;36mAlbertLayerGroup.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, output_attentions, output_hidden_states)\u001b[0m\n\u001b[0;32m    433\u001b[0m layer_attentions \u001b[38;5;241m=\u001b[39m ()\n\u001b[0;32m    435\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m layer_index, albert_layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39malbert_layers):\n\u001b[1;32m--> 436\u001b[0m     layer_output \u001b[38;5;241m=\u001b[39m \u001b[43malbert_layer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m[\u001b[49m\u001b[43mlayer_index\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    437\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m layer_output[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    439\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m output_attentions:\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\transformers\\models\\albert\\modeling_albert.py:399\u001b[0m, in \u001b[0;36mAlbertLayer.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, output_attentions, output_hidden_states)\u001b[0m\n\u001b[0;32m    391\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[0;32m    392\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    393\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    397\u001b[0m     output_hidden_states: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m    398\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor, torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m--> 399\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    401\u001b[0m     ffn_output \u001b[38;5;241m=\u001b[39m apply_chunking_to_forward(\n\u001b[0;32m    402\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mff_chunk,\n\u001b[0;32m    403\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchunk_size_feed_forward,\n\u001b[0;32m    404\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mseq_len_dim,\n\u001b[0;32m    405\u001b[0m         attention_output[\u001b[38;5;241m0\u001b[39m],\n\u001b[0;32m    406\u001b[0m     )\n\u001b[0;32m    407\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfull_layer_layer_norm(ffn_output \u001b[38;5;241m+\u001b[39m attention_output[\u001b[38;5;241m0\u001b[39m])\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\transformers\\models\\albert\\modeling_albert.py:327\u001b[0m, in \u001b[0;36mAlbertAttention.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, output_attentions)\u001b[0m\n\u001b[0;32m    325\u001b[0m mixed_query_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquery(hidden_states)\n\u001b[0;32m    326\u001b[0m mixed_key_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkey(hidden_states)\n\u001b[1;32m--> 327\u001b[0m mixed_value_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    329\u001b[0m query_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtranspose_for_scores(mixed_query_layer)\n\u001b[0;32m    330\u001b[0m key_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtranspose_for_scores(mixed_key_layer)\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m   1106\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1107\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1109\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1111\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32mC:\\apps\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\linear.py:103\u001b[0m, in \u001b[0;36mLinear.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    102\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 103\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 36.00 MiB (GPU 0; 6.00 GiB total capacity; 5.14 GiB already allocated; 0 bytes free; 5.30 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "### https://www.sbert.net/docs/package_reference/losses.html#batchsemihardtripletloss\n",
    "## supports 1 sentence/label pair\n",
    "train_dataset = SentencesDataset(train_examples, model2)\n",
    "train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=256)\n",
    "train_loss = losses.BatchSemiHardTripletLoss(model=model2)\n",
    "\n",
    "#Tune the model\n",
    "model2.fit(train_objectives=[(train_dataloader, train_loss)], epochs=2, warmup_steps=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uxfcIQ85-nlQ"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of 3-CAH_finetune.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "030325446b8e4ede91020a4536f91064": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0b2804c196ef4f369aa469a3080cb930": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b17a4f8b6fca414286b47a65e60d1254",
       "IPY_MODEL_6b8fcd36f1bb4a22981f0162ae538d7f",
       "IPY_MODEL_62bb978e5a3b4a9cab47a8856c3633d0"
      ],
      "layout": "IPY_MODEL_2941cf6f1bf745b7a8518ce2b1e8aaf7"
     }
    },
    "12f0eead030b48e79d989ab53ae75f5a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "24c0db6a9c2146bea1b658c7245044aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ef2b365a6ece4bdb8d9cb84b6e593047",
      "max": 9786,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_35d4592d40f344ae92e27bd0bdc0f129",
      "value": 9786
     }
    },
    "2941cf6f1bf745b7a8518ce2b1e8aaf7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "35d4592d40f344ae92e27bd0bdc0f129": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "3f969476da554298b79fb91c4ec6e08f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9952f2e5f62d402eadd85c0c17d5d8c2",
      "placeholder": "​",
      "style": "IPY_MODEL_12f0eead030b48e79d989ab53ae75f5a",
      "value": "Batches: 100%"
     }
    },
    "62bb978e5a3b4a9cab47a8856c3633d0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b597b302d1d9405fbf9181d8a62e3944",
      "placeholder": "​",
      "style": "IPY_MODEL_a5e9540e89ec4bee85fb45235c2fc640",
      "value": " 2447/2447 [05:01&lt;00:00, 21.96it/s]"
     }
    },
    "6b8fcd36f1bb4a22981f0162ae538d7f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8c44dbe9d1c74aae894b1ff86ae69c9b",
      "max": 2447,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_75d0eb084e1b429b9375196c6069c55d",
      "value": 2447
     }
    },
    "75d0eb084e1b429b9375196c6069c55d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "82841c43fda44080b6debfd95e425b0f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d54fadb082c64e1e868ed9b21bdb4698",
      "placeholder": "​",
      "style": "IPY_MODEL_9933a11516d64e00a16d8793fbb2ac04",
      "value": " 9786/9786 [19:57&lt;00:00, 22.67it/s]"
     }
    },
    "82cc1edd77f047b084a5f0c0b17a4bb1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8c44dbe9d1c74aae894b1ff86ae69c9b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9933a11516d64e00a16d8793fbb2ac04": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9952f2e5f62d402eadd85c0c17d5d8c2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9f284c9e3c364f778c41b8c7e0ba3045": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a5e9540e89ec4bee85fb45235c2fc640": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b17a4f8b6fca414286b47a65e60d1254": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9f284c9e3c364f778c41b8c7e0ba3045",
      "placeholder": "​",
      "style": "IPY_MODEL_030325446b8e4ede91020a4536f91064",
      "value": "Batches: 100%"
     }
    },
    "b597b302d1d9405fbf9181d8a62e3944": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cfea03270fe64a44b54cde507f629eaf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3f969476da554298b79fb91c4ec6e08f",
       "IPY_MODEL_24c0db6a9c2146bea1b658c7245044aa",
       "IPY_MODEL_82841c43fda44080b6debfd95e425b0f"
      ],
      "layout": "IPY_MODEL_82cc1edd77f047b084a5f0c0b17a4bb1"
     }
    },
    "d54fadb082c64e1e868ed9b21bdb4698": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ef2b365a6ece4bdb8d9cb84b6e593047": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}