docs/notebooks/fork-of-mind-bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-18T06:15:17.623964500Z",
     "start_time": "2024-01-18T06:15:09.158831300Z"
    },
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-12-30T07:25:38.885885Z",
     "iopub.status.busy": "2023-12-30T07:25:38.885106Z",
     "iopub.status.idle": "2023-12-30T07:25:39.536428Z",
     "shell.execute_reply": "2023-12-30T07:25:39.535194Z",
     "shell.execute_reply.started": "2023-12-30T07:25:38.885846Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Reading dataset\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../models/news.tsv', header=None, sep=\"\\t\")\n",
    "\n",
    "df.columns=['id',\n",
    "\"category\",\n",
    "\"subcategory\",\n",
    "\"title\",\n",
    "\"abstract\",\n",
    "\"url\",\n",
    "\"title_entities\",\n",
    "\"abstract_entities \"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:25:41.955777Z",
     "iopub.status.busy": "2023-12-30T07:25:41.955419Z",
     "iopub.status.idle": "2023-12-30T07:25:42.006481Z",
     "shell.execute_reply": "2023-12-30T07:25:42.005725Z",
     "shell.execute_reply.started": "2023-12-30T07:25:41.955749Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Mengambil 1000 data acak\n",
    "df_sampled = df.sample(n=1000, random_state=42)  # Anda dapat mengganti nilai random_state sesuai kebutuhan\n",
    "\n",
    "# Menyimpan dataset baru\n",
    "df_sampled.to_csv('sampled_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:25:44.680081Z",
     "iopub.status.busy": "2023-12-30T07:25:44.679707Z",
     "iopub.status.idle": "2023-12-30T07:25:44.694509Z",
     "shell.execute_reply": "2023-12-30T07:25:44.693501Z",
     "shell.execute_reply.started": "2023-12-30T07:25:44.680052Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>subcategory</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>url</th>\n",
       "      <th>title_entities</th>\n",
       "      <th>abstract_entities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>36810</th>\n",
       "      <td>N7433</td>\n",
       "      <td>news</td>\n",
       "      <td>newsus</td>\n",
       "      <td>The 3 best Greek spots in Aurora</td>\n",
       "      <td>Hoodline crunched the numbers to find the top ...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWHLdc.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24447</th>\n",
       "      <td>N43326</td>\n",
       "      <td>sports</td>\n",
       "      <td>more_sports</td>\n",
       "      <td>Officials investigating stabbing death of Alex...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJD8WV.html</td>\n",
       "      <td>[{\"Label\": \"Prison\", \"Type\": \"C\", \"WikidataId\"...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23187</th>\n",
       "      <td>N45745</td>\n",
       "      <td>finance</td>\n",
       "      <td>finance-top-stocks</td>\n",
       "      <td>SmileDirectClub tanks on California bill, brin...</td>\n",
       "      <td>Shares of SmileDirectClub sank to a new all-ti...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAILU2e.html</td>\n",
       "      <td>[{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...</td>\n",
       "      <td>[{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22077</th>\n",
       "      <td>N5234</td>\n",
       "      <td>travel</td>\n",
       "      <td>traveltips</td>\n",
       "      <td>The world's 50 most reliable airlines</td>\n",
       "      <td>No one wants to kick off their holiday with de...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJub6N.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{\"Label\": \"Get Going\", \"Type\": \"N\", \"Wikidata...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32260</th>\n",
       "      <td>N58530</td>\n",
       "      <td>sports</td>\n",
       "      <td>basketball_ncaa</td>\n",
       "      <td>High school football: Breaking down the Class ...</td>\n",
       "      <td>The Class 3A football playoffs begin Friday ac...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWAERc.html</td>\n",
       "      <td>[{\"Label\": \"South African Class 3A 4-8-2\", \"Ty...</td>\n",
       "      <td>[{\"Label\": \"Heritage Hall School\", \"Type\": \"F\"...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id category         subcategory  \\\n",
       "36810   N7433     news              newsus   \n",
       "24447  N43326   sports         more_sports   \n",
       "23187  N45745  finance  finance-top-stocks   \n",
       "22077   N5234   travel          traveltips   \n",
       "32260  N58530   sports     basketball_ncaa   \n",
       "\n",
       "                                                   title  \\\n",
       "36810                   The 3 best Greek spots in Aurora   \n",
       "24447  Officials investigating stabbing death of Alex...   \n",
       "23187  SmileDirectClub tanks on California bill, brin...   \n",
       "22077              The world's 50 most reliable airlines   \n",
       "32260  High school football: Breaking down the Class ...   \n",
       "\n",
       "                                                abstract  \\\n",
       "36810  Hoodline crunched the numbers to find the top ...   \n",
       "24447                                                NaN   \n",
       "23187  Shares of SmileDirectClub sank to a new all-ti...   \n",
       "22077  No one wants to kick off their holiday with de...   \n",
       "32260  The Class 3A football playoffs begin Friday ac...   \n",
       "\n",
       "                                                 url  \\\n",
       "36810  https://assets.msn.com/labs/mind/BBWHLdc.html   \n",
       "24447  https://assets.msn.com/labs/mind/AAJD8WV.html   \n",
       "23187  https://assets.msn.com/labs/mind/AAILU2e.html   \n",
       "22077  https://assets.msn.com/labs/mind/AAJub6N.html   \n",
       "32260  https://assets.msn.com/labs/mind/BBWAERc.html   \n",
       "\n",
       "                                          title_entities  \\\n",
       "36810                                                 []   \n",
       "24447  [{\"Label\": \"Prison\", \"Type\": \"C\", \"WikidataId\"...   \n",
       "23187  [{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...   \n",
       "22077                                                 []   \n",
       "32260  [{\"Label\": \"South African Class 3A 4-8-2\", \"Ty...   \n",
       "\n",
       "                                      abstract_entities   \n",
       "36810                                                 []  \n",
       "24447                                                 []  \n",
       "23187  [{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...  \n",
       "22077  [{\"Label\": \"Get Going\", \"Type\": \"N\", \"Wikidata...  \n",
       "32260  [{\"Label\": \"Heritage Hall School\", \"Type\": \"F\"...  "
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sampled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:25:49.216483Z",
     "iopub.status.busy": "2023-12-30T07:25:49.215982Z",
     "iopub.status.idle": "2023-12-30T07:25:49.235857Z",
     "shell.execute_reply": "2023-12-30T07:25:49.234816Z",
     "shell.execute_reply.started": "2023-12-30T07:25:49.216443Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "df_sampled['corpus'] = df_sampled[['title', 'abstract']].apply(lambda x: ' '.join(map(str, x)), axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-18T06:53:53.957205900Z",
     "start_time": "2024-01-18T06:53:22.755721600Z"
    },
    "execution": {
     "iopub.execute_input": "2023-12-30T07:25:53.992044Z",
     "iopub.status.busy": "2023-12-30T07:25:53.991156Z",
     "iopub.status.idle": "2023-12-30T07:25:53.996246Z",
     "shell.execute_reply": "2023-12-30T07:25:53.995332Z",
     "shell.execute_reply.started": "2023-12-30T07:25:53.992010Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:25:57.554305Z",
     "iopub.status.busy": "2023-12-30T07:25:57.553660Z",
     "iopub.status.idle": "2023-12-30T07:26:08.118066Z",
     "shell.execute_reply": "2023-12-30T07:26:08.117081Z",
     "shell.execute_reply.started": "2023-12-30T07:25:57.554250Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "preprocessor = hub.KerasLayer(\n",
    "    \"../models/bert_preprocessor\",)\n",
    "encoder = hub.KerasLayer(\n",
    "    \"../models/bert_encoder\",\n",
    "    trainable=True)\n",
    "\n",
    "\n",
    "def get_bert_embeddings(text, preprocessor, encoder):\n",
    "    text_input = tf.keras.layers.Input(shape=(), dtype=tf.string)\n",
    "    encoder_inputs = preprocessor(text_input)\n",
    "    outputs = encoder(encoder_inputs)\n",
    "    embedding_model = tf.keras.Model(text_input, outputs['pooled_output'])\n",
    "    embedding_model.compile(optimizer='adam', loss='mse')\n",
    "    sentences = tf.constant([text])\n",
    "    emb = embedding_model(sentences).numpy()\n",
    "    print(emb)\n",
    "    return emb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_kg_hide-output": true,
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2023-12-30T07:26:08.120911Z",
     "iopub.status.busy": "2023-12-30T07:26:08.120124Z",
     "iopub.status.idle": "2023-12-30T07:28:38.200389Z",
     "shell.execute_reply": "2023-12-30T07:28:38.199413Z",
     "shell.execute_reply.started": "2023-12-30T07:26:08.120875Z"
    },
    "jupyter": {
     "outputs_hidden": true
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "df_sampled['encodings'] = df_sampled['corpus'].apply(lambda x: get_bert_embeddings(x, preprocessor, encoder))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:32:26.339606Z",
     "iopub.status.busy": "2023-12-30T07:32:26.339205Z",
     "iopub.status.idle": "2023-12-30T07:32:26.362967Z",
     "shell.execute_reply": "2023-12-30T07:32:26.361984Z",
     "shell.execute_reply.started": "2023-12-30T07:32:26.339575Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>subcategory</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>url</th>\n",
       "      <th>title_entities</th>\n",
       "      <th>abstract_entities</th>\n",
       "      <th>corpus</th>\n",
       "      <th>encodings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>36810</th>\n",
       "      <td>N7433</td>\n",
       "      <td>news</td>\n",
       "      <td>newsus</td>\n",
       "      <td>The 3 best Greek spots in Aurora</td>\n",
       "      <td>Hoodline crunched the numbers to find the top ...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWHLdc.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>The 3 best Greek spots in Aurora Hoodline crun...</td>\n",
       "      <td>[[-0.88768613, 0.9936917, -0.012753431, -0.927...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24447</th>\n",
       "      <td>N43326</td>\n",
       "      <td>sports</td>\n",
       "      <td>more_sports</td>\n",
       "      <td>Officials investigating stabbing death of Alex...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJD8WV.html</td>\n",
       "      <td>[{\"Label\": \"Prison\", \"Type\": \"C\", \"WikidataId\"...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Officials investigating stabbing death of Alex...</td>\n",
       "      <td>[[-0.71360165, 0.9078096, 0.031543188, -0.9471...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23187</th>\n",
       "      <td>N45745</td>\n",
       "      <td>finance</td>\n",
       "      <td>finance-top-stocks</td>\n",
       "      <td>SmileDirectClub tanks on California bill, brin...</td>\n",
       "      <td>Shares of SmileDirectClub sank to a new all-ti...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAILU2e.html</td>\n",
       "      <td>[{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...</td>\n",
       "      <td>[{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...</td>\n",
       "      <td>SmileDirectClub tanks on California bill, brin...</td>\n",
       "      <td>[[-0.69691885, 0.99604875, 0.055066865, -0.982...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22077</th>\n",
       "      <td>N5234</td>\n",
       "      <td>travel</td>\n",
       "      <td>traveltips</td>\n",
       "      <td>The world's 50 most reliable airlines</td>\n",
       "      <td>No one wants to kick off their holiday with de...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJub6N.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{\"Label\": \"Get Going\", \"Type\": \"N\", \"Wikidata...</td>\n",
       "      <td>The world's 50 most reliable airlines No one w...</td>\n",
       "      <td>[[-0.5948372, 0.9595088, 0.006537994, -0.86174...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32260</th>\n",
       "      <td>N58530</td>\n",
       "      <td>sports</td>\n",
       "      <td>basketball_ncaa</td>\n",
       "      <td>High school football: Breaking down the Class ...</td>\n",
       "      <td>The Class 3A football playoffs begin Friday ac...</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWAERc.html</td>\n",
       "      <td>[{\"Label\": \"South African Class 3A 4-8-2\", \"Ty...</td>\n",
       "      <td>[{\"Label\": \"Heritage Hall School\", \"Type\": \"F\"...</td>\n",
       "      <td>High school football: Breaking down the Class ...</td>\n",
       "      <td>[[-0.7885393, 0.91186565, 0.008350249, -0.9251...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id category         subcategory  \\\n",
       "36810   N7433     news              newsus   \n",
       "24447  N43326   sports         more_sports   \n",
       "23187  N45745  finance  finance-top-stocks   \n",
       "22077   N5234   travel          traveltips   \n",
       "32260  N58530   sports     basketball_ncaa   \n",
       "\n",
       "                                                   title  \\\n",
       "36810                   The 3 best Greek spots in Aurora   \n",
       "24447  Officials investigating stabbing death of Alex...   \n",
       "23187  SmileDirectClub tanks on California bill, brin...   \n",
       "22077              The world's 50 most reliable airlines   \n",
       "32260  High school football: Breaking down the Class ...   \n",
       "\n",
       "                                                abstract  \\\n",
       "36810  Hoodline crunched the numbers to find the top ...   \n",
       "24447                                                NaN   \n",
       "23187  Shares of SmileDirectClub sank to a new all-ti...   \n",
       "22077  No one wants to kick off their holiday with de...   \n",
       "32260  The Class 3A football playoffs begin Friday ac...   \n",
       "\n",
       "                                                 url  \\\n",
       "36810  https://assets.msn.com/labs/mind/BBWHLdc.html   \n",
       "24447  https://assets.msn.com/labs/mind/AAJD8WV.html   \n",
       "23187  https://assets.msn.com/labs/mind/AAILU2e.html   \n",
       "22077  https://assets.msn.com/labs/mind/AAJub6N.html   \n",
       "32260  https://assets.msn.com/labs/mind/BBWAERc.html   \n",
       "\n",
       "                                          title_entities  \\\n",
       "36810                                                 []   \n",
       "24447  [{\"Label\": \"Prison\", \"Type\": \"C\", \"WikidataId\"...   \n",
       "23187  [{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...   \n",
       "22077                                                 []   \n",
       "32260  [{\"Label\": \"South African Class 3A 4-8-2\", \"Ty...   \n",
       "\n",
       "                                      abstract_entities   \\\n",
       "36810                                                 []   \n",
       "24447                                                 []   \n",
       "23187  [{\"Label\": \"SmileDirectClub\", \"Type\": \"N\", \"Wi...   \n",
       "22077  [{\"Label\": \"Get Going\", \"Type\": \"N\", \"Wikidata...   \n",
       "32260  [{\"Label\": \"Heritage Hall School\", \"Type\": \"F\"...   \n",
       "\n",
       "                                                  corpus  \\\n",
       "36810  The 3 best Greek spots in Aurora Hoodline crun...   \n",
       "24447  Officials investigating stabbing death of Alex...   \n",
       "23187  SmileDirectClub tanks on California bill, brin...   \n",
       "22077  The world's 50 most reliable airlines No one w...   \n",
       "32260  High school football: Breaking down the Class ...   \n",
       "\n",
       "                                               encodings  \n",
       "36810  [[-0.88768613, 0.9936917, -0.012753431, -0.927...  \n",
       "24447  [[-0.71360165, 0.9078096, 0.031543188, -0.9471...  \n",
       "23187  [[-0.69691885, 0.99604875, 0.055066865, -0.982...  \n",
       "22077  [[-0.5948372, 0.9595088, 0.006537994, -0.86174...  \n",
       "32260  [[-0.7885393, 0.91186565, 0.008350249, -0.9251...  "
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sampled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:32:34.019688Z",
     "iopub.status.busy": "2023-12-30T07:32:34.018839Z",
     "iopub.status.idle": "2023-12-30T07:32:34.030682Z",
     "shell.execute_reply": "2023-12-30T07:32:34.029790Z",
     "shell.execute_reply.started": "2023-12-30T07:32:34.019654Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 1000 entries, 36810 to 18618\n",
      "Data columns (total 10 columns):\n",
      " #   Column              Non-Null Count  Dtype \n",
      "---  ------              --------------  ----- \n",
      " 0   id                  1000 non-null   object\n",
      " 1   category            1000 non-null   object\n",
      " 2   subcategory         1000 non-null   object\n",
      " 3   title               1000 non-null   object\n",
      " 4   abstract            954 non-null    object\n",
      " 5   url                 1000 non-null   object\n",
      " 6   title_entities      1000 non-null   object\n",
      " 7   abstract_entities   1000 non-null   object\n",
      " 8   corpus              1000 non-null   object\n",
      " 9   encodings           1000 non-null   object\n",
      "dtypes: object(10)\n",
      "memory usage: 85.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df_sampled.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:39:58.102238Z",
     "iopub.status.busy": "2023-12-30T07:39:58.101213Z",
     "iopub.status.idle": "2023-12-30T07:40:04.331529Z",
     "shell.execute_reply": "2023-12-30T07:40:04.330765Z",
     "shell.execute_reply.started": "2023-12-30T07:39:58.102192Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " foodball\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.8863599   0.9589162  -0.03594382 -0.98949367 -0.9884794  -0.3459113\n",
      "  -0.8367941   0.08099159  0.86258954 -0.9053912  -0.9294881   0.30097944\n",
      "   0.77174187  0.98076636 -0.00978982  0.9272214  -0.9963344   0.02808348\n",
      "  -0.8501529   0.91581637 -0.86578435 -0.7428405  -0.7980818   0.9396636\n",
      "  -0.98776275  0.8862722  -0.17799507 -0.99847865 -0.8128746  -0.98041326\n",
      "   0.9649863   0.9654179   0.99694437  0.9335473   0.5981218  -0.01160499\n",
      "  -0.9704506  -0.66767526  0.469542   -0.23900217 -0.97021896 -0.8280334\n",
      "   0.14835533  0.99970907 -0.75166255  0.04118969 -0.7701593  -0.76283234\n",
      "   0.9723812   0.9948737  -0.72955745 -0.97511554  0.44438463  0.42426747\n",
      "  -0.02824351 -0.90026426  0.22260025  0.17239353 -0.83641386  0.95690715\n",
      "  -0.9987625  -0.930684   -0.10681489 -0.07957909 -0.7871317   0.9832969\n",
      "   0.04694906 -0.9642784  -0.02843418 -0.8033848  -0.9945915   0.04037812\n",
      "  -0.78310126 -0.03518964  0.99929035  0.23422825  0.00609834 -0.19585001\n",
      "   0.5542175   0.10316495  0.7932266   0.9977634  -0.7029955   0.96635395\n",
      "   0.96902764  0.78006715  0.11064091  0.11341745 -0.63541657 -0.1828126\n",
      "   0.9428148   0.7974125  -0.7842087   0.8968533  -0.5456611  -0.64469945\n",
      "  -0.20090573  0.9078176  -0.15128574 -0.03065263  0.23450385 -0.86945456\n",
      "  -0.84031886 -0.9962156   0.962109    0.97905934  0.7981063  -0.93095326\n",
      "  -0.8025591  -0.9605223   0.99886066  0.3118939   0.5991014   0.93824303\n",
      "  -0.8643421  -0.98446673 -0.9034726  -0.45142004  0.3672646  -0.80402863\n",
      "  -0.81666714 -0.9670842  -0.78982925  0.00408366 -0.14829007  0.99061495\n",
      "   0.18194455  0.07944082]]\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from sklearn import metrics\n",
    "\n",
    "def preprocess_text():\n",
    "  text = input()\n",
    "  text = text.lower()\n",
    "  text = re.sub('[^A-Za-z0-9]+', ' ', text)\n",
    "  return text\n",
    "  \n",
    "query_text = preprocess_text()\n",
    "query_encoding = get_bert_embeddings(query_text, preprocessor, encoder)\n",
    "\n",
    "df_sampled['similarity_score'] = df_sampled['encodings'].apply(lambda x: metrics.pairwise.cosine_similarity(x, query_encoding)[0][0])\n",
    "df_results = df_sampled.sort_values(by=['similarity_score'], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:40:06.739550Z",
     "iopub.status.busy": "2023-12-30T07:40:06.738732Z",
     "iopub.status.idle": "2023-12-30T07:40:06.764590Z",
     "shell.execute_reply": "2023-12-30T07:40:06.763587Z",
     "shell.execute_reply.started": "2023-12-30T07:40:06.739506Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>subcategory</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>url</th>\n",
       "      <th>title_entities</th>\n",
       "      <th>abstract_entities</th>\n",
       "      <th>corpus</th>\n",
       "      <th>encodings</th>\n",
       "      <th>similarity_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33344</th>\n",
       "      <td>N38838</td>\n",
       "      <td>sports</td>\n",
       "      <td>more_sports</td>\n",
       "      <td>Fantasy Football: Start 'em, sit 'em for Week 11</td>\n",
       "      <td>Questionable fantasy football advice every week</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWH0Id.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>Fantasy Football: Start 'em, sit 'em for Week ...</td>\n",
       "      <td>[[-0.8269854, 0.92667043, -0.009696479, -0.908...</td>\n",
       "      <td>0.938967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5297</th>\n",
       "      <td>N57287</td>\n",
       "      <td>sports</td>\n",
       "      <td>football_ncaa</td>\n",
       "      <td>Pigskin Poll: How will the Cleveland Browns fi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJATJG.html</td>\n",
       "      <td>[{\"Label\": \"Cleveland Browns\", \"Type\": \"O\", \"W...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Pigskin Poll: How will the Cleveland Browns fi...</td>\n",
       "      <td>[[-0.925933, 0.93152684, -0.029294569, -0.9728...</td>\n",
       "      <td>0.933846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7294</th>\n",
       "      <td>N5217</td>\n",
       "      <td>sports</td>\n",
       "      <td>football_nfl</td>\n",
       "      <td>Patriots release Josh Gordon: Should Dolphins ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJF1VI.html</td>\n",
       "      <td>[{\"Label\": \"Josh Gordon\", \"Type\": \"P\", \"Wikida...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Patriots release Josh Gordon: Should Dolphins ...</td>\n",
       "      <td>[[-0.9573617, 0.9234477, -0.028576326, -0.8975...</td>\n",
       "      <td>0.932811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40462</th>\n",
       "      <td>N36757</td>\n",
       "      <td>sports</td>\n",
       "      <td>basketball_nba</td>\n",
       "      <td>De'Aaron Fox suffers ankle sprain, per report</td>\n",
       "      <td>Ugh</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWBDYl.html</td>\n",
       "      <td>[{\"Label\": \"De'Aaron Fox\", \"Type\": \"P\", \"Wikid...</td>\n",
       "      <td>[]</td>\n",
       "      <td>De'Aaron Fox suffers ankle sprain, per report Ugh</td>\n",
       "      <td>[[-0.8546091, 0.9091533, 0.036464766, -0.96581...</td>\n",
       "      <td>0.931382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31201</th>\n",
       "      <td>N57415</td>\n",
       "      <td>lifestyle</td>\n",
       "      <td>lifestylelovesex</td>\n",
       "      <td>17 Winter Date Ideas That Will Melt Your Cold,...</td>\n",
       "      <td>*Calls cuff right now*</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBPSDiZ.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>17 Winter Date Ideas That Will Melt Your Cold,...</td>\n",
       "      <td>[[-0.98657054, 0.9769207, -0.19236892, -0.9693...</td>\n",
       "      <td>0.929504</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id   category       subcategory  \\\n",
       "33344  N38838     sports       more_sports   \n",
       "5297   N57287     sports     football_ncaa   \n",
       "7294    N5217     sports      football_nfl   \n",
       "40462  N36757     sports    basketball_nba   \n",
       "31201  N57415  lifestyle  lifestylelovesex   \n",
       "\n",
       "                                                   title  \\\n",
       "33344   Fantasy Football: Start 'em, sit 'em for Week 11   \n",
       "5297   Pigskin Poll: How will the Cleveland Browns fi...   \n",
       "7294   Patriots release Josh Gordon: Should Dolphins ...   \n",
       "40462      De'Aaron Fox suffers ankle sprain, per report   \n",
       "31201  17 Winter Date Ideas That Will Melt Your Cold,...   \n",
       "\n",
       "                                              abstract  \\\n",
       "33344  Questionable fantasy football advice every week   \n",
       "5297                                               NaN   \n",
       "7294                                               NaN   \n",
       "40462                                              Ugh   \n",
       "31201                           *Calls cuff right now*   \n",
       "\n",
       "                                                 url  \\\n",
       "33344  https://assets.msn.com/labs/mind/BBWH0Id.html   \n",
       "5297   https://assets.msn.com/labs/mind/AAJATJG.html   \n",
       "7294   https://assets.msn.com/labs/mind/AAJF1VI.html   \n",
       "40462  https://assets.msn.com/labs/mind/BBWBDYl.html   \n",
       "31201  https://assets.msn.com/labs/mind/BBPSDiZ.html   \n",
       "\n",
       "                                          title_entities abstract_entities   \\\n",
       "33344                                                 []                 []   \n",
       "5297   [{\"Label\": \"Cleveland Browns\", \"Type\": \"O\", \"W...                 []   \n",
       "7294   [{\"Label\": \"Josh Gordon\", \"Type\": \"P\", \"Wikida...                 []   \n",
       "40462  [{\"Label\": \"De'Aaron Fox\", \"Type\": \"P\", \"Wikid...                 []   \n",
       "31201                                                 []                 []   \n",
       "\n",
       "                                                  corpus  \\\n",
       "33344  Fantasy Football: Start 'em, sit 'em for Week ...   \n",
       "5297   Pigskin Poll: How will the Cleveland Browns fi...   \n",
       "7294   Patriots release Josh Gordon: Should Dolphins ...   \n",
       "40462  De'Aaron Fox suffers ankle sprain, per report Ugh   \n",
       "31201  17 Winter Date Ideas That Will Melt Your Cold,...   \n",
       "\n",
       "                                               encodings  similarity_score  \n",
       "33344  [[-0.8269854, 0.92667043, -0.009696479, -0.908...          0.938967  \n",
       "5297   [[-0.925933, 0.93152684, -0.029294569, -0.9728...          0.933846  \n",
       "7294   [[-0.9573617, 0.9234477, -0.028576326, -0.8975...          0.932811  \n",
       "40462  [[-0.8546091, 0.9091533, 0.036464766, -0.96581...          0.931382  \n",
       "31201  [[-0.98657054, 0.9769207, -0.19236892, -0.9693...          0.929504  "
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:33:08.783416Z",
     "iopub.status.busy": "2023-12-30T07:33:08.783051Z",
     "iopub.status.idle": "2023-12-30T07:33:08.794660Z",
     "shell.execute_reply": "2023-12-30T07:33:08.793659Z",
     "shell.execute_reply.started": "2023-12-30T07:33:08.783389Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Reciprocal Rank (MRR): 0.007485470860550343\n"
     ]
    }
   ],
   "source": [
    "def hitung_mrr(df_results):\n",
    "    indeks_relevan = df_results[df_results['similarity_score'] > 0]['similarity_score'].index\n",
    "    reciprocal_ranks = []\n",
    "\n",
    "    for idx in indeks_relevan:\n",
    "        peringkat = df_results.index.get_loc(idx) + 1\n",
    "        reciprocal_rank = 1 / peringkat\n",
    "        reciprocal_ranks.append(reciprocal_rank)\n",
    "\n",
    "    if reciprocal_ranks:\n",
    "        mrr = sum(reciprocal_ranks) / len(reciprocal_ranks)\n",
    "        return mrr\n",
    "    else:\n",
    "        return 0.0\n",
    "\n",
    "skor_mrr = hitung_mrr(df_results)\n",
    "print(f\"Mean Reciprocal Rank (MRR): {skor_mrr}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:40:52.290721Z",
     "iopub.status.busy": "2023-12-30T07:40:52.290320Z",
     "iopub.status.idle": "2023-12-30T07:40:52.306298Z",
     "shell.execute_reply": "2023-12-30T07:40:52.305219Z",
     "shell.execute_reply.started": "2023-12-30T07:40:52.290691Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36810    [[-0.88768613, 0.9936917, -0.012753431, -0.927...\n",
       "24447    [[-0.71360165, 0.9078096, 0.031543188, -0.9471...\n",
       "23187    [[-0.69691885, 0.99604875, 0.055066865, -0.982...\n",
       "22077    [[-0.5948372, 0.9595088, 0.006537994, -0.86174...\n",
       "32260    [[-0.7885393, 0.91186565, 0.008350249, -0.9251...\n",
       "                               ...                        \n",
       "22605    [[-0.81792533, 0.98405755, -0.09422249, -0.929...\n",
       "6234     [[-0.8103939, 0.9874413, 0.15309896, -0.889150...\n",
       "21726    [[-0.9600377, 0.98605675, -0.0405233, -0.97683...\n",
       "21504    [[-0.38789156, 0.9881241, -0.017219935, -0.910...\n",
       "18618    [[-0.8083892, 0.9927196, 0.08153918, -0.886835...\n",
       "Name: encodings, Length: 1000, dtype: object"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sampled.encodings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T09:28:29.194824Z",
     "iopub.status.busy": "2023-12-30T09:28:29.194570Z",
     "iopub.status.idle": "2023-12-30T09:28:41.745667Z",
     "shell.execute_reply": "2023-12-30T09:28:41.744600Z",
     "shell.execute_reply.started": "2023-12-30T09:28:29.194801Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: tensorflow\n",
      "Version: 2.13.0\n",
      "Summary: TensorFlow is an open source machine learning framework for everyone.\n",
      "Home-page: https://www.tensorflow.org/\n",
      "Author: Google Inc.\n",
      "Author-email: packages@tensorflow.org\n",
      "License: Apache 2.0\n",
      "Location: /opt/conda/lib/python3.10/site-packages\n",
      "Requires: absl-py, astunparse, flatbuffers, gast, google-pasta, grpcio, h5py, keras, libclang, numpy, opt-einsum, packaging, protobuf, setuptools, six, tensorboard, tensorflow-estimator, tensorflow-io-gcs-filesystem, termcolor, typing-extensions, wrapt\n",
      "Required-by: explainable-ai-sdk, tensorflow-cloud, tensorflow-decision-forests, tensorflow-serving-api, tensorflow-text, tensorflowjs, witwidget\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "!pip show tensorflow\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:40:57.947581Z",
     "iopub.status.busy": "2023-12-30T07:40:57.947171Z",
     "iopub.status.idle": "2023-12-30T07:40:59.350214Z",
     "shell.execute_reply": "2023-12-30T07:40:59.349021Z",
     "shell.execute_reply.started": "2023-12-30T07:40:57.947550Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "df_sampled.to_csv('output_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:41:00.288817Z",
     "iopub.status.busy": "2023-12-30T07:41:00.287962Z",
     "iopub.status.idle": "2023-12-30T07:41:00.328981Z",
     "shell.execute_reply": "2023-12-30T07:41:00.328222Z",
     "shell.execute_reply.started": "2023-12-30T07:41:00.288784Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('/kaggle/working/output_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:41:02.335336Z",
     "iopub.status.busy": "2023-12-30T07:41:02.334951Z",
     "iopub.status.idle": "2023-12-30T07:41:02.343429Z",
     "shell.execute_reply": "2023-12-30T07:41:02.342416Z",
     "shell.execute_reply.started": "2023-12-30T07:41:02.335306Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      [[-0.88768613  0.9936917  -0.01275343 -0.92733...\n",
       "1      [[-7.13601649e-01  9.07809615e-01  3.15431878e...\n",
       "2      [[-0.69691885  0.99604875  0.05506686 -0.98216...\n",
       "3      [[-0.5948372   0.9595088   0.00653799 -0.86174...\n",
       "4      [[-0.7885393   0.91186565  0.00835025 -0.92517...\n",
       "                             ...                        \n",
       "995    [[-0.81792533  0.98405755 -0.09422249 -0.92918...\n",
       "996    [[-0.8103939   0.9874413   0.15309896 -0.88915...\n",
       "997    [[-0.9600377   0.98605675 -0.0405233  -0.97683...\n",
       "998    [[-0.38789156  0.9881241  -0.01721993 -0.91077...\n",
       "999    [[-8.08389187e-01  9.92719591e-01  8.15391764e...\n",
       "Name: encodings, Length: 1000, dtype: object"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:41:05.889863Z",
     "iopub.status.busy": "2023-12-30T07:41:05.889495Z",
     "iopub.status.idle": "2023-12-30T07:41:05.902285Z",
     "shell.execute_reply": "2023-12-30T07:41:05.901325Z",
     "shell.execute_reply.started": "2023-12-30T07:41:05.889838Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 11 columns):\n",
      " #   Column              Non-Null Count  Dtype  \n",
      "---  ------              --------------  -----  \n",
      " 0   id                  1000 non-null   object \n",
      " 1   category            1000 non-null   object \n",
      " 2   subcategory         1000 non-null   object \n",
      " 3   title               1000 non-null   object \n",
      " 4   abstract            954 non-null    object \n",
      " 5   url                 1000 non-null   object \n",
      " 6   title_entities      1000 non-null   object \n",
      " 7   abstract_entities   1000 non-null   object \n",
      " 8   corpus              1000 non-null   object \n",
      " 9   encodings           1000 non-null   object \n",
      " 10  similarity_score    1000 non-null   float64\n",
      "dtypes: float64(1), object(10)\n",
      "memory usage: 86.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:44:46.559242Z",
     "iopub.status.busy": "2023-12-30T07:44:46.558643Z",
     "iopub.status.idle": "2023-12-30T07:44:46.597068Z",
     "shell.execute_reply": "2023-12-30T07:44:46.596111Z",
     "shell.execute_reply.started": "2023-12-30T07:44:46.559209Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('/kaggle/working/output_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:44:46.911416Z",
     "iopub.status.busy": "2023-12-30T07:44:46.910691Z",
     "iopub.status.idle": "2023-12-30T07:44:46.958723Z",
     "shell.execute_reply": "2023-12-30T07:44:46.957771Z",
     "shell.execute_reply.started": "2023-12-30T07:44:46.911383Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "df['encodings'] = df['encodings'].apply(lambda x: np.fromstring(x.strip('[]'), sep=' '))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:45:20.788841Z",
     "iopub.status.busy": "2023-12-30T07:45:20.788441Z",
     "iopub.status.idle": "2023-12-30T07:45:24.042804Z",
     "shell.execute_reply": "2023-12-30T07:45:24.041767Z",
     "shell.execute_reply.started": "2023-12-30T07:45:20.788810Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " foodball\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.8863599   0.9589162  -0.03594382 -0.98949367 -0.9884794  -0.3459113\n",
      "  -0.8367941   0.08099159  0.86258954 -0.9053912  -0.9294881   0.30097944\n",
      "   0.77174187  0.98076636 -0.00978982  0.9272214  -0.9963344   0.02808348\n",
      "  -0.8501529   0.91581637 -0.86578435 -0.7428405  -0.7980818   0.9396636\n",
      "  -0.98776275  0.8862722  -0.17799507 -0.99847865 -0.8128746  -0.98041326\n",
      "   0.9649863   0.9654179   0.99694437  0.9335473   0.5981218  -0.01160499\n",
      "  -0.9704506  -0.66767526  0.469542   -0.23900217 -0.97021896 -0.8280334\n",
      "   0.14835533  0.99970907 -0.75166255  0.04118969 -0.7701593  -0.76283234\n",
      "   0.9723812   0.9948737  -0.72955745 -0.97511554  0.44438463  0.42426747\n",
      "  -0.02824351 -0.90026426  0.22260025  0.17239353 -0.83641386  0.95690715\n",
      "  -0.9987625  -0.930684   -0.10681489 -0.07957909 -0.7871317   0.9832969\n",
      "   0.04694906 -0.9642784  -0.02843418 -0.8033848  -0.9945915   0.04037812\n",
      "  -0.78310126 -0.03518964  0.99929035  0.23422825  0.00609834 -0.19585001\n",
      "   0.5542175   0.10316495  0.7932266   0.9977634  -0.7029955   0.96635395\n",
      "   0.96902764  0.78006715  0.11064091  0.11341745 -0.63541657 -0.1828126\n",
      "   0.9428148   0.7974125  -0.7842087   0.8968533  -0.5456611  -0.64469945\n",
      "  -0.20090573  0.9078176  -0.15128574 -0.03065263  0.23450385 -0.86945456\n",
      "  -0.84031886 -0.9962156   0.962109    0.97905934  0.7981063  -0.93095326\n",
      "  -0.8025591  -0.9605223   0.99886066  0.3118939   0.5991014   0.93824303\n",
      "  -0.8643421  -0.98446673 -0.9034726  -0.45142004  0.3672646  -0.80402863\n",
      "  -0.81666714 -0.9670842  -0.78982925  0.00408366 -0.14829007  0.99061495\n",
      "   0.18194455  0.07944082]]\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from sklearn import metrics\n",
    "\n",
    "def preprocess_text():\n",
    "  text = input()\n",
    "  text = text.lower()\n",
    "  text = re.sub('[^A-Za-z0-9]+', ' ', text)\n",
    "  return text\n",
    "  \n",
    "query_text = preprocess_text()\n",
    "query_encoding = get_bert_embeddings(query_text, preprocessor, encoder)\n",
    "\n",
    "df['similarity_score'] = df['encodings'].apply(lambda x: metrics.pairwise.cosine_similarity(x.reshape(1, -1), query_encoding.reshape(1, -1))[0][0])\n",
    "df_results = df.sort_values(by=['similarity_score'], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T06:55:43.766613Z",
     "iopub.status.busy": "2023-12-30T06:55:43.765870Z",
     "iopub.status.idle": "2023-12-30T06:55:43.772554Z",
     "shell.execute_reply": "2023-12-30T06:55:43.771681Z",
     "shell.execute_reply.started": "2023-12-30T06:55:43.766581Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[[-7.13601649e-01 9.07809615e-01 3.15431878e-02 -9.47138727e-01 -9.52696681e-01 -4.15526032e-01 -7.95670629e-01 -4.93095815e-02 9.17906225e-01 -9.19885993e-01 -7.52385736e-01 6.70269251e-01 1.07117556e-01 9.88221347e-01 -5.00168577e-02 9.62122262e-01 -9.97160256e-01 2.46147096e-01 -9.15880799e-01 5.44235826e-01 -8.70210350e-01 -4.15583313e-01 -9.54319179e-01 8.82264972e-01 -9.93970454e-01 9.25391078e-01 7.78293073e-01 -9.89409089e-01 3.11202884e-01 -9.79765892e-01 8.71633410e-01 8.78490865e-01 9.92934287e-01 7.27473021e-01 -1.23550981e-01 -1.25989512e-01 -9.97903883e-01 -1.28183916e-01 8.39935541e-01 1.47923842e-01 -8.49081635e-01 -9.47495639e-01 3.97986639e-03 9.95476484e-01 -7.65434325e-01 -4.60150272e-01 -8.96885216e-01 -4.63224351e-01 9.42373991e-01 9.56772149e-01 -9.76478815e-01 -8.15842688e-01 7.63803422e-01 -4.75163817e-01 -1.19623639e-01 -6.27890944e-01 1.08177364e-01 -8.06506258e-03 -9.81450319e-01 9.72026527e-01 -9.95928884e-01 -6.97246492e-02 5.37368536e-01 -2.68909074e-02 -4.58419236e-04 8.78219068e-01 7.71321505e-02 -9.13244903e-01 -3.67249809e-02 -8.99613678e-01 -9.83195305e-01 4.15641814e-03 -9.51224327e-01 -9.68520865e-02 9.99333620e-01 -5.00953555e-01 9.55823511e-02 -1.41267702e-02 1.92127883e-01 7.50279650e-02 7.54430532e-01 9.98244226e-01 -8.13326538e-01 9.33356822e-01 6.86049342e-01 9.05134976e-01 7.56313354e-02 4.10502069e-02 -3.09943467e-01 -1.49210408e-01 8.17655742e-01 9.18912947e-01 -9.61662114e-01 5.38369536e-01 -6.39263391e-02 -9.69015121e-01 8.36148635e-02 7.68704355e-01 -8.53047892e-02 -1.16031803e-02 -4.22307104e-01 -4.98149782e-01 -7.95411348e-01 -9.94664848e-01 9.56931472e-01 9.97495949e-01 2.05630049e-01 -8.84599149e-01 3.86949122e-01 -8.36145341e-01 9.95876431e-01 6.27484977e-01 2.40575433e-01 6.98803544e-01 -5.53883791e-01 -9.94238377e-01 -9.54716921e-01 -5.62251866e-01 2.64733791e-01 -5.10053098e-01 -8.18359375e-01 -4.86190170e-01 2.40690842e-01 5.10892682e-02 6.87513828e-01 9.79677498e-01 -5.79140604e-01 1.29730478e-01]]'"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.encodings[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-30T07:45:39.466253Z",
     "iopub.status.busy": "2023-12-30T07:45:39.465889Z",
     "iopub.status.idle": "2023-12-30T07:45:39.489542Z",
     "shell.execute_reply": "2023-12-30T07:45:39.488337Z",
     "shell.execute_reply.started": "2023-12-30T07:45:39.466223Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>subcategory</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>url</th>\n",
       "      <th>title_entities</th>\n",
       "      <th>abstract_entities</th>\n",
       "      <th>corpus</th>\n",
       "      <th>encodings</th>\n",
       "      <th>similarity_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>555</th>\n",
       "      <td>N38838</td>\n",
       "      <td>sports</td>\n",
       "      <td>more_sports</td>\n",
       "      <td>Fantasy Football: Start 'em, sit 'em for Week 11</td>\n",
       "      <td>Questionable fantasy football advice every week</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWH0Id.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>Fantasy Football: Start 'em, sit 'em for Week ...</td>\n",
       "      <td>[-0.8269854, 0.92667043, -0.00969648, -0.90889...</td>\n",
       "      <td>0.938967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>N57287</td>\n",
       "      <td>sports</td>\n",
       "      <td>football_ncaa</td>\n",
       "      <td>Pigskin Poll: How will the Cleveland Browns fi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJATJG.html</td>\n",
       "      <td>[{\"Label\": \"Cleveland Browns\", \"Type\": \"O\", \"W...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Pigskin Poll: How will the Cleveland Browns fi...</td>\n",
       "      <td>[-0.925933, 0.93152684, -0.02929457, -0.972871...</td>\n",
       "      <td>0.933846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>N5217</td>\n",
       "      <td>sports</td>\n",
       "      <td>football_nfl</td>\n",
       "      <td>Patriots release Josh Gordon: Should Dolphins ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://assets.msn.com/labs/mind/AAJF1VI.html</td>\n",
       "      <td>[{\"Label\": \"Josh Gordon\", \"Type\": \"P\", \"Wikida...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Patriots release Josh Gordon: Should Dolphins ...</td>\n",
       "      <td>[-0.9573617, 0.9234477, -0.02857633, -0.897563...</td>\n",
       "      <td>0.932811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554</th>\n",
       "      <td>N36757</td>\n",
       "      <td>sports</td>\n",
       "      <td>basketball_nba</td>\n",
       "      <td>De'Aaron Fox suffers ankle sprain, per report</td>\n",
       "      <td>Ugh</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBWBDYl.html</td>\n",
       "      <td>[{\"Label\": \"De'Aaron Fox\", \"Type\": \"P\", \"Wikid...</td>\n",
       "      <td>[]</td>\n",
       "      <td>De'Aaron Fox suffers ankle sprain, per report Ugh</td>\n",
       "      <td>[-0.8546091, 0.9091533, 0.03646477, -0.9658108...</td>\n",
       "      <td>0.931382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>N57415</td>\n",
       "      <td>lifestyle</td>\n",
       "      <td>lifestylelovesex</td>\n",
       "      <td>17 Winter Date Ideas That Will Melt Your Cold,...</td>\n",
       "      <td>*Calls cuff right now*</td>\n",
       "      <td>https://assets.msn.com/labs/mind/BBPSDiZ.html</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>17 Winter Date Ideas That Will Melt Your Cold,...</td>\n",
       "      <td>[-0.98657054, 0.9769207, -0.19236892, -0.96933...</td>\n",
       "      <td>0.929503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id   category       subcategory  \\\n",
       "555  N38838     sports       more_sports   \n",
       "650  N57287     sports     football_ncaa   \n",
       "634   N5217     sports      football_nfl   \n",
       "554  N36757     sports    basketball_nba   \n",
       "61   N57415  lifestyle  lifestylelovesex   \n",
       "\n",
       "                                                 title  \\\n",
       "555   Fantasy Football: Start 'em, sit 'em for Week 11   \n",
       "650  Pigskin Poll: How will the Cleveland Browns fi...   \n",
       "634  Patriots release Josh Gordon: Should Dolphins ...   \n",
       "554      De'Aaron Fox suffers ankle sprain, per report   \n",
       "61   17 Winter Date Ideas That Will Melt Your Cold,...   \n",
       "\n",
       "                                            abstract  \\\n",
       "555  Questionable fantasy football advice every week   \n",
       "650                                              NaN   \n",
       "634                                              NaN   \n",
       "554                                              Ugh   \n",
       "61                            *Calls cuff right now*   \n",
       "\n",
       "                                               url  \\\n",
       "555  https://assets.msn.com/labs/mind/BBWH0Id.html   \n",
       "650  https://assets.msn.com/labs/mind/AAJATJG.html   \n",
       "634  https://assets.msn.com/labs/mind/AAJF1VI.html   \n",
       "554  https://assets.msn.com/labs/mind/BBWBDYl.html   \n",
       "61   https://assets.msn.com/labs/mind/BBPSDiZ.html   \n",
       "\n",
       "                                        title_entities abstract_entities   \\\n",
       "555                                                 []                 []   \n",
       "650  [{\"Label\": \"Cleveland Browns\", \"Type\": \"O\", \"W...                 []   \n",
       "634  [{\"Label\": \"Josh Gordon\", \"Type\": \"P\", \"Wikida...                 []   \n",
       "554  [{\"Label\": \"De'Aaron Fox\", \"Type\": \"P\", \"Wikid...                 []   \n",
       "61                                                  []                 []   \n",
       "\n",
       "                                                corpus  \\\n",
       "555  Fantasy Football: Start 'em, sit 'em for Week ...   \n",
       "650  Pigskin Poll: How will the Cleveland Browns fi...   \n",
       "634  Patriots release Josh Gordon: Should Dolphins ...   \n",
       "554  De'Aaron Fox suffers ankle sprain, per report Ugh   \n",
       "61   17 Winter Date Ideas That Will Melt Your Cold,...   \n",
       "\n",
       "                                             encodings  similarity_score  \n",
       "555  [-0.8269854, 0.92667043, -0.00969648, -0.90889...          0.938967  \n",
       "650  [-0.925933, 0.93152684, -0.02929457, -0.972871...          0.933846  \n",
       "634  [-0.9573617, 0.9234477, -0.02857633, -0.897563...          0.932811  \n",
       "554  [-0.8546091, 0.9091533, 0.03646477, -0.9658108...          0.931382  \n",
       "61   [-0.98657054, 0.9769207, -0.19236892, -0.96933...          0.929503  "
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T07:51:25.242378Z",
     "iopub.status.busy": "2023-12-19T07:51:25.241547Z",
     "iopub.status.idle": "2023-12-19T07:51:49.936196Z",
     "shell.execute_reply": "2023-12-19T07:51:49.935397Z",
     "shell.execute_reply.started": "2023-12-19T07:51:25.242344Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "preprocessor_path = \"/kaggle/working/bert_preprocessor\"\n",
    "encoder_path = \"/kaggle/working/bert_encoder\"\n",
    "tf.saved_model.save(preprocessor, preprocessor_path)\n",
    "tf.saved_model.save(encoder, encoder_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T07:53:26.135784Z",
     "iopub.status.busy": "2023-12-19T07:53:26.13458Z",
     "iopub.status.idle": "2023-12-19T07:54:32.262701Z",
     "shell.execute_reply": "2023-12-19T07:54:32.261789Z",
     "shell.execute_reply.started": "2023-12-19T07:53:26.135738Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('/kaggle/working/similarity_results.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T08:01:09.743647Z",
     "iopub.status.busy": "2023-12-19T08:01:09.742843Z",
     "iopub.status.idle": "2023-12-19T08:01:11.131818Z",
     "shell.execute_reply": "2023-12-19T08:01:11.130686Z",
     "shell.execute_reply.started": "2023-12-19T08:01:09.74361Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "!kaggle kernels output -q notebook-id -p /kaggle/working\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T08:02:22.184625Z",
     "iopub.status.busy": "2023-12-19T08:02:22.183659Z",
     "iopub.status.idle": "2023-12-19T08:02:32.819698Z",
     "shell.execute_reply": "2023-12-19T08:02:32.81873Z",
     "shell.execute_reply.started": "2023-12-19T08:02:22.184588Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "!zip -r output.zip /kaggle/working/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-19T08:20:54.657179Z",
     "iopub.status.busy": "2023-12-19T08:20:54.656717Z",
     "iopub.status.idle": "2023-12-19T08:20:54.663397Z",
     "shell.execute_reply": "2023-12-19T08:20:54.662453Z",
     "shell.execute_reply.started": "2023-12-19T08:20:54.657145Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Menyimpan path model preprocessor dan encoder ke dalam file\n",
    "with open('/kaggle/working/model_paths.txt', 'w') as file:\n",
    "    file.write(f\"Preprocessor: {preprocessor_path}\\n\")\n",
    "    file.write(f\"Encoder: {encoder_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [
    {
     "datasetId": 1049650,
     "sourceId": 2565112,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4190479,
     "sourceId": 7236259,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4239140,
     "sourceId": 7306237,
     "sourceType": "datasetVersion"
    },
    {
     "modelInstanceId": 1899,
     "sourceId": 2622,
     "sourceType": "modelInstanceVersion"
    },
    {
     "modelInstanceId": 2180,
     "sourceId": 2938,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30627,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}