Doc2Vec.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Doc2Vec.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOAi22Orm123Wixx0m4mjfw",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shashank-m/covid_mining_papers/blob/master/Doc2Vec.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dkjIIK9BSPHw",
        "colab_type": "text"
      },
      "source": [
        "### Imports and setting up files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfkX688ax5ll",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import codecs\n",
        "from tqdm import tqdm\n",
        "import spacy\n",
        "import matplotlib.pyplot as plt\n",
        "import nltk\n",
        "from nltk.util import ngrams\n",
        "import re\n",
        "import json\n",
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0vQSWFbzFq8s",
        "colab_type": "code",
        "outputId": "af30bdd2-890b-47be-9867-9ed68cc0b964",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "nltk.download('punkt')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EIgG-tkXu0W2",
        "colab_type": "code",
        "outputId": "95a7bf20-bf67-4e71-8039-188014d1e30e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.autograd import Variable\n",
        "from torch.nn import functional as F\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "torch.manual_seed(1)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7fab888ee490>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WFUYtfQ2vWvm",
        "colab_type": "code",
        "outputId": "15bbed1c-c10a-4db5-aa9c-43722de7d9a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\n",
        "is_cuda = torch.cuda.is_available()\n",
        "if is_cuda:\n",
        "    device = torch.device(\"cuda\")\n",
        "    print(\"GPU is available\")\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "    print(\"GPU not available, CPU used\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU is available\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PPmdPOarxzpl",
        "colab_type": "code",
        "outputId": "9a70eb2a-574b-46e4-c428-3ae4f87c36d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GStG0DA1Mr-M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filenames=os.listdir(\"/content/drive/My Drive/covid_nlp/2020-03-13/biorxiv_medrxiv/biorxiv_medrxiv\")\n",
        "biorxiv=\"/content/drive/My Drive/covid_nlp/2020-03-13/biorxiv_medrxiv/biorxiv_medrxiv/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8LM5Ha1sRScR",
        "colab_type": "text"
      },
      "source": [
        "## ***Preprocessing of abstract***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BPXYtEecsWI_",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RQ02m2Z-Q8vp",
        "colab": {}
      },
      "source": [
        "# all_files=[json.load(open(biorxiv+filename, 'rb')) for filename in filenames]\n",
        "all_files=[]\n",
        "for filename in filenames:\n",
        "  all_files.append(json.load(open(biorxiv+filename, 'rb')))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ZAI-MrH-ezyv",
        "colab": {}
      },
      "source": [
        "def preprocess(abstract):\n",
        "  abstract=abstract.lower()\n",
        "  abstract=re.sub(r'[,-?<>:!_+=#$()%^{}|~@;\\'\"*&[\\]]',' ',abstract) #remove punctuation.\n",
        "  abstract=re.sub(r'\\d+',' ',abstract) # remove digits.\n",
        "  abstract=re.sub(r'\\s+',' ',abstract).strip() # replace multiple white spaces with single whiite space.\n",
        "  return abstract"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K748TrxnLL6H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cleaned_abstracts=[]\n",
        "for i,f in enumerate(all_files):\n",
        "  if f['abstract']: # checks if abstract exists or not.\n",
        "    abstract=''\n",
        "\n",
        "    for dic in f['abstract']:\n",
        "      abstract+=dic['text']\n",
        "      abstract+='\\n'\n",
        "    abstract=preprocess(abstract)\n",
        "\n",
        "    if abstract!='' and len(re.findall(r'\\S+',abstract))>=5: # rejects abstracts which are empty or less than 5 words. \n",
        "      cleaned_abstracts.append((abstract,i)) # appending a tuple of abstract and paper number. \n",
        "no_docs=len(cleaned_abstracts)     "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iXA_8NeXgAAo",
        "colab_type": "text"
      },
      "source": [
        "### **Create train and test data for Doc2Vec model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfGyU0WN9yrY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "v=''\n",
        "for doc,i in cleaned_abstracts:\n",
        "  v+=doc\n",
        "v=v.split()\n",
        "vocab=set(v)\n",
        "vocab_size=len(vocab)\n",
        "word_to_idx={}\n",
        "for i,word in enumerate(vocab):\n",
        "  word_to_idx[word]=i\n",
        "word_to_idx['<UNK>']=vocab_size  \n",
        "vocab_size+=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IBqaYH-z0AqV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def get_train_tensor(cleaned_abstracts,c):\n",
        "  \"\"\"\n",
        "  Cleaned abstracts is a list of tuples where each tuple contains the paper number(from alll_files list)\n",
        "  and the cleaned abstract.\n",
        "  \"\"\"\n",
        "  total_input=[]\n",
        "  total_lable=[]\n",
        "  i=0\n",
        "  test_word_loc=int((c-1)/2)\n",
        "  for doc,file_num in (cleaned_abstracts):\n",
        "    if doc:\n",
        "      train_data=[]\n",
        "      all_tokens=re.findall(r'\\S+',doc)\n",
        "      penta_gram=ngrams(all_tokens,c) # c is window size for ngram. c=n.\n",
        "      \n",
        "      for window in penta_gram:\n",
        "        window=list(window)\n",
        "        test_word=window[test_word_loc]\n",
        "        train_words=window[:test_word_loc]+window[(test_word_loc+1):]\n",
        "        train_words.append(i)\n",
        "        train_data.append((train_words,test_word)) \n",
        "\n",
        "      context_data=[]\n",
        "      target_data=[]\n",
        "\n",
        "      for context,target in train_data:\n",
        "\n",
        "        inputs=[]\n",
        "        for word in context[:-1]:\n",
        "          try:\n",
        "            inputs.append(word_to_idx[word])\n",
        "          except KeyError:\n",
        "            inputs.append(word_to_idx['<UNK>'])\n",
        "        inputs.append(i)   # i is the document number.   \n",
        "        inputs.append(file_num) # file_num is the file in biorxiv folder. They may not be same as some files were removed due to lack of abstract.\n",
        "        inputs=torch.LongTensor(inputs).view(1,-1)\n",
        "        target_vector=torch.LongTensor([word_to_idx[target]])\n",
        "        context_data.append(inputs)\n",
        "        target_data.append(target_vector)\n",
        "\n",
        "      try:\n",
        "        X_train=torch.cat(context_data,0)\n",
        "        X_train_label=torch.cat(target_data,0)\n",
        "        total_input.append(X_train)\n",
        "        total_lable.append(X_train_label)\n",
        "        i+=1\n",
        "      except RuntimeError:\n",
        "        print(i,file_num)\n",
        "        pass\n",
        "  return total_input,total_lable\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0jl1EReSKsc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "window=5\n",
        "total_input,total_lable=get_train_tensor(cleaned_abstracts,window)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XCt0DotFLw5C",
        "colab_type": "text"
      },
      "source": [
        "**Last column of X_train contains file_num and last but one contains document no.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "doiANoKt9Kqe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train=torch.cat(total_input,0) # put all training_data into a single tensor so that it can be batched by Dataloader."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d55HfFViAMmp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train_label=torch.cat(total_lable,0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liGM4_aPBJIi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size=64\n",
        "training=TensorDataset(X_train,X_train_label)\n",
        "train_loader=DataLoader(training,batch_size=batch_size,drop_last=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d1vFYaM5Ph-O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1b8afa7c-6119-487c-aed3-98b7c1521e1a"
      },
      "source": [
        "X_train.shape"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([212592, 6])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0pLSxnUHRtSF",
        "colab_type": "text"
      },
      "source": [
        "### Doc2Vec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Q6EcYdVNy1o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class doc2vec(nn.Module):\n",
        "  def __init__(self,vocab_size,embed_dim,context,hidden_dim,batch_size,no_docs):\n",
        "    super(doc2vec,self).__init__()\n",
        "    self.ndocs=no_docs\n",
        "    self.bs=batch_size\n",
        "    self.c=context\n",
        "\n",
        "    self.w_embeddings=nn.Embedding(vocab_size,embed_dim)\n",
        "    self.d_embeddings=nn.Embedding(no_docs,embed_dim)\n",
        "\n",
        "    self.linear1=nn.Linear(embed_dim,hidden_dim)\n",
        "    self.linear2=nn.Linear(hidden_dim,vocab_size)\n",
        "    \n",
        "  def forward(self,inputs):\n",
        "      \n",
        "      w_embeds=self.w_embeddings(inputs[:,:4])\n",
        "      d_embeds=self.d_embeddings(inputs[:,4]).unsqueeze(1)\n",
        "      combined=torch.cat((w_embeds,d_embeds),1)\n",
        "      a_1=F.relu(self.linear1(combined).sum(axis=1)/(self.c+1))\n",
        "      out=self.linear2(a_1)\n",
        "      return out\n",
        "\n",
        "embed_dim=300\n",
        "context=window-1\n",
        "hidden_dim=50\n",
        "model=doc2vec(vocab_size,embed_dim,context,hidden_dim,batch_size,no_docs)\n",
        "criterion=nn.CrossEntropyLoss()\n",
        "lr=1e-3\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "\n",
        "model.state_dict\n",
        "if is_cuda:\n",
        "  model.cuda()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uUyW5XC2sKxP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5f22ab4c-375e-483a-c0b2-d974087e75a9"
      },
      "source": [
        "model.state_dict()['d_embeddings.weight'].shape\n"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([693, 300])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Q7-Amdneu2v",
        "colab_type": "text"
      },
      "source": [
        "**Note that here training data is generated from all abstracts\\.We run the Doc2Vec model over 3 epochs and it looks like the model is learning something as the loss decreases every epoch. If we can overfit on this small data that means out training is not an issue.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B-sN2ccJYreu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def train_infer_loop(epochs,loader,model):\n",
        "  count=0\n",
        "  for j in range(epochs):\n",
        "    # train_loss=[]\n",
        "    for i,(x,y) in enumerate(loader):\n",
        "      count+=1\n",
        "      model.zero_grad()\n",
        "      if(is_cuda):\n",
        "        x, y = x.cuda(), y.cuda()\n",
        "      out=model(x)\n",
        "\n",
        "      loss=criterion(out,y)\n",
        "      loss.backward()\n",
        "\n",
        "      # train_loss.append(loss.item())\n",
        "\n",
        "      optimizer.step()\n",
        "      # if i%400==0:\n",
        "      #   print('epoch {},batch={},running train loss={}'.format(j+1,i,np.average(train_loss))) \n",
        "    print(\"epoch {} , loss= {}\".format(j+1,loss.item()))   "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8irxJ-H1kEE3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "eac38976-1c27-49a6-e160-51617d732ada"
      },
      "source": [
        "train_infer_loop(10,train_loader,model)"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1 , loss= 6.309681415557861\n",
            "epoch 2 , loss= 5.852118015289307\n",
            "epoch 3 , loss= 5.512955188751221\n",
            "epoch 4 , loss= 5.181860446929932\n",
            "epoch 5 , loss= 4.9008259773254395\n",
            "epoch 6 , loss= 4.63656759262085\n",
            "epoch 7 , loss= 4.324553966522217\n",
            "epoch 8 , loss= 4.156632900238037\n",
            "epoch 9 , loss= 3.985248565673828\n",
            "epoch 10 , loss= 3.8501224517822266\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2uDHF3-tBxFr",
        "colab_type": "text"
      },
      "source": [
        "**Inference stage.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kf6NZb2CnGkW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "f82c2dae-3193-4844-89c3-6237e09678b8"
      },
      "source": [
        "model.linear1.weight"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Parameter containing:\n",
              "tensor([[ 0.2298,  0.1238,  0.3842,  ...,  0.1489,  0.1802,  0.0185],\n",
              "        [-0.1405,  0.0601,  0.0624,  ...,  0.1533, -0.1157, -0.0396],\n",
              "        [ 0.3770, -0.4081, -0.3381,  ...,  0.4333,  0.0632, -0.2612],\n",
              "        ...,\n",
              "        [-0.2157,  0.0762,  0.1830,  ..., -0.1227, -0.2109, -0.3166],\n",
              "        [-0.2073,  0.2350,  0.1430,  ..., -0.0952, -0.2026, -0.0797],\n",
              "        [-0.0725, -0.0123,  0.0772,  ...,  0.0566,  0.0515,  0.4032]],\n",
              "       device='cuda:0', requires_grad=True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WIizBjDnymwR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "d9ce3fa0-7d73-4cc9-9d4b-2bd3af16d70d"
      },
      "source": [
        "for key in model.state_dict():\n",
        "  print(key)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "w_embeddings.weight\n",
            "d_embeddings.weight\n",
            "linear1.weight\n",
            "linear1.bias\n",
            "linear2.weight\n",
            "linear2.bias\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbJCuMdAUPkh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e9f50eec-155a-4544-d529-7a2cca108707"
      },
      "source": [
        "model.state_dict()['d_embeddings.weight'][0]"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 7.6642e-01, -5.8539e-01, -1.5511e+00,  3.9555e-01, -1.4272e+00,\n",
              "        -1.9348e+00,  2.9453e-01, -6.1008e-01, -3.7993e-01, -2.5643e+00,\n",
              "        -2.7140e+00,  3.5350e-01, -8.2012e-01,  1.8017e+00,  3.0188e-01,\n",
              "        -6.7503e-01, -9.3303e-01, -5.0716e-01, -3.6577e+00,  1.5420e-02,\n",
              "        -6.3998e-01,  2.0770e+00, -2.9984e-01, -8.8807e-01,  3.5561e-01,\n",
              "        -1.4543e+00, -7.8796e-01,  1.0569e-01, -1.0248e+00, -5.9145e-01,\n",
              "        -6.3116e-01, -4.5057e-02,  3.8976e-01,  2.0394e-01, -1.1785e+00,\n",
              "         1.0208e+00,  1.2712e+00,  6.4129e-01, -1.4899e+00,  1.1558e+00,\n",
              "         6.4827e-01, -8.0238e-02,  7.5821e-02, -6.7544e-01,  2.1301e+00,\n",
              "        -7.1728e-01, -9.9774e-01,  7.2381e-01, -2.3320e+00,  6.0348e-01,\n",
              "        -1.2715e+00, -9.0010e-01, -3.4766e-01,  6.5404e-02,  6.1175e-01,\n",
              "        -8.3246e-01,  1.5995e-02,  8.3488e-01, -1.3093e+00,  4.7996e-01,\n",
              "         5.2170e-01,  2.2406e-01, -1.8375e+00,  1.6658e+00, -6.7962e-01,\n",
              "         2.5200e-01,  3.5381e-01,  8.0617e-01, -2.2310e-01,  7.8387e-01,\n",
              "         1.6937e+00, -1.1702e-02,  1.2610e+00, -1.9609e+00, -2.5008e-01,\n",
              "        -1.7032e-01, -2.5505e-01, -8.5157e-01,  1.2179e+00,  2.9196e-01,\n",
              "        -4.3136e-01, -1.9271e+00, -8.8689e-02,  2.0342e-01,  2.0102e-01,\n",
              "        -1.2004e+00,  1.8346e+00,  1.4678e-01, -1.5704e-01,  2.4147e+00,\n",
              "         1.5794e+00, -8.1343e-01,  1.3439e+00, -6.9394e-01,  1.9694e-01,\n",
              "        -3.9145e-01, -8.7434e-01, -1.1474e+00, -4.3776e-01,  3.3920e-02,\n",
              "        -1.5422e+00,  1.9280e-01, -1.8947e-01,  1.3469e+00, -5.2797e-01,\n",
              "        -7.9490e-01,  2.3063e+00,  1.0073e-02,  1.0048e+00, -5.4308e-01,\n",
              "         5.9146e-01,  6.1885e-01, -6.2840e-01, -1.6218e+00, -1.3222e+00,\n",
              "        -3.2032e-01, -1.0144e+00, -7.5949e-01, -5.0913e-01,  2.7953e-01,\n",
              "         1.7754e+00,  8.1838e-01, -9.0506e-02,  2.2353e-01, -1.0330e+00,\n",
              "         2.0235e+00, -1.4548e+00, -2.8035e-02, -5.2031e-01,  2.3907e-01,\n",
              "         2.9530e-01, -1.5815e+00,  2.8300e-02, -9.4471e-01,  6.6845e-01,\n",
              "         1.3006e-01,  3.6984e-01, -1.2364e+00,  6.3703e-01, -4.3449e-01,\n",
              "        -4.0202e-02,  9.3638e-01, -1.0289e+00, -7.8278e-01, -1.2721e-01,\n",
              "         2.3924e-01,  1.4762e+00,  1.9131e-01, -1.2154e+00, -9.2715e-01,\n",
              "        -4.3525e-01,  2.8345e-01,  9.6093e-01, -1.3137e+00, -3.0043e-01,\n",
              "        -8.7688e-01, -1.0865e+00, -6.9670e-01,  6.0397e-01,  1.9383e+00,\n",
              "        -1.8217e-01, -4.0976e-01, -2.5056e-01, -7.9182e-01,  9.8328e-01,\n",
              "        -3.9818e-01, -3.2178e-03,  5.6416e-01,  1.1869e+00,  1.1932e+00,\n",
              "        -1.9157e+00,  6.0474e-01, -1.7703e-01,  1.4195e+00,  2.3594e+00,\n",
              "         1.3982e+00, -1.9496e+00, -1.3129e+00, -9.4401e-01, -4.7342e-01,\n",
              "        -9.0117e-01,  1.3427e+00,  8.3357e-01,  5.5676e-01, -9.8444e-01,\n",
              "        -1.0905e+00, -8.8977e-01, -9.4355e-01, -3.9074e-01, -2.4144e-01,\n",
              "        -1.4777e+00, -2.6619e-01,  1.7415e-01,  2.8204e-01, -2.0486e-01,\n",
              "        -1.3503e+00, -6.6132e-01, -3.6765e-01,  1.3479e-01,  2.4111e-01,\n",
              "        -1.2335e+00,  7.6537e-01, -4.7348e-01,  2.3056e-01, -2.6774e-01,\n",
              "        -5.6074e-01, -6.0534e-01,  1.6628e+00, -1.3599e+00, -2.2093e+00,\n",
              "         3.3973e-02, -1.9466e+00,  5.9325e-01,  2.3255e-01,  2.7028e-01,\n",
              "         1.0493e+00, -5.0865e-01, -6.4963e-01, -4.7599e-01,  1.4566e-02,\n",
              "         3.7317e-01,  3.2199e-01,  7.8164e-01,  6.6465e-01,  1.4214e+00,\n",
              "        -5.7500e-01, -7.4075e-01,  6.4843e-01, -6.0601e-01, -2.0807e+00,\n",
              "        -1.2265e+00,  4.8036e-01, -7.7683e-01, -5.6120e-01,  8.0328e-01,\n",
              "        -1.2776e+00, -8.8956e-01, -1.2131e+00,  1.4979e+00, -4.7832e-01,\n",
              "        -2.0440e-01,  1.2393e+00,  1.0670e+00,  3.9944e-01,  7.5598e-01,\n",
              "        -2.8491e+00, -2.2609e+00, -9.9950e-01,  4.4233e-01,  2.4867e-01,\n",
              "         8.7370e-01, -6.5260e-01,  2.2663e-01, -6.0966e-01, -1.5359e+00,\n",
              "         2.4569e-01,  9.9607e-01, -4.6534e-01,  1.9994e+00,  5.7452e-01,\n",
              "         1.0086e+00,  1.4037e+00,  1.0638e+00,  5.6912e-01,  1.7502e+00,\n",
              "        -5.1997e-01,  4.1683e-01, -1.2151e+00,  5.0638e-02,  8.4310e-01,\n",
              "         3.1131e-01,  8.0834e-01, -1.1666e+00,  1.2405e+00,  2.7593e-02,\n",
              "         1.4243e+00, -1.0961e+00,  6.4939e-01,  5.1070e-02,  1.7797e-01,\n",
              "        -7.7434e-01, -9.1596e-01,  3.7486e-02, -9.3170e-01,  1.3539e+00,\n",
              "        -1.1393e+00,  1.4034e+00,  1.0779e-02,  1.0455e+00, -3.6527e-02,\n",
              "        -8.1508e-01, -8.6794e-01, -5.5947e-01,  4.7297e-01, -1.3143e+00,\n",
              "        -1.3525e+00,  6.8503e-01, -1.7091e+00,  2.1459e-01,  1.1057e+00],\n",
              "       device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kPk1vuU7hSDq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "query='Efforts targeted at a universal coronavirus vaccine.'\n",
        "query=preprocess(query)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wf_bQ-0booIs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "infer_data=[(query,0)]\n",
        "infer_window=3\n",
        "data,label=get_train_tensor(infer_data,infer_window)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P_grXr8tXjD7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_input=torch.cat(data,0)\n",
        "x_label=torch.cat(label,0)\n",
        "\n",
        "batch_size=1\n",
        "inference=TensorDataset(x_input,x_label)\n",
        "infer_loader=DataLoader(inference,batch_size=batch_size,drop_last=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ih6n7QtOSJkD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d9ef50ab-a3cc-4782-eeef-8ec013ca0309"
      },
      "source": [
        "for x,y in infer_loader:\n",
        "  print(x[0][2]+693)\n",
        "  break"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(693)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zy0Ytr2mhMdW",
        "colab": {}
      },
      "source": [
        "class infer(nn.Module):\n",
        "  def __init__(self,vocab_size,embed_dim,context,hidden_dim,batch_size,no_docs,dic):\n",
        "    super(infer,self).__init__()\n",
        "    self.ndocs=no_docs\n",
        "    self.bs=batch_size\n",
        "    self.c=context\n",
        "\n",
        "    self.w_embeddings=nn.Embedding(vocab_size,embed_dim)\n",
        "\n",
        "    self.w_embeddings.weight=nn.Parameter(dic['w_embeddings.weight'])\n",
        "    self.w_embeddings.weight.requires_grad=False\n",
        "\n",
        "\n",
        "    self.linear1=nn.Linear(embed_dim,hidden_dim)\n",
        "    self.linear2=nn.Linear(hidden_dim,vocab_size)\n",
        "\n",
        "    self.linear1.weight=nn.Parameter(dic['linear1.weight'])\n",
        "    self.linear1.bias=nn.Parameter(dic['linear1.bias'])\n",
        "    self.linear2.weight=nn.Parameter(dic['linear2.weight'])\n",
        "    self.linear2.bias=nn.Parameter(dic['linear2.bias'])\n",
        "\n",
        "    l=[self.linear1,self.linear2]\n",
        "    for layer in l:\n",
        "      layer.weight.requires_grad=False\n",
        "      layer.bias.requires_grad=False\n",
        "    # old=dic['d_embeddings.weight']\n",
        "    # add=torch.randn(1,300).cuda()\n",
        "\n",
        "    self.d_embeddings=nn.Embedding(no_docs,embed_dim)\n",
        "    # self.d_embeddings.weight=nn.Parameter(torch.cat((old,add),0))\n",
        "    # self.d_embeddings.weight=nn.Parameter(dic['d_embeddings.weight'])\n",
        "  def forward(self,inputs):\n",
        "    w_embeds=self.w_embeddings(inputs[0,:self.c])\n",
        "    d_embeds=self.d_embeddings(inputs[0,self.c]).unsqueeze(0)\n",
        "    combined=torch.cat((w_embeds,d_embeds),0)\n",
        "    a_1=F.relu(self.linear1(combined).sum(axis=0)/(self.c+1))\n",
        "    out=self.linear2(a_1).unsqueeze(0)\n",
        "\n",
        "    return out\n",
        "    \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G-LWQasfgpoZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "hidden_dim=50\n",
        "infer_context=infer_window-1\n",
        "batch_size=1\n",
        "no_docs=1\n",
        "embed_dim=300\n",
        "model_infer=infer(vocab_size,embed_dim,infer_context,hidden_dim,batch_size,no_docs,model.state_dict())\n",
        "criterion=nn.CrossEntropyLoss()\n",
        "lr=1e-3\n",
        "optimizer = torch.optim.Adam(model_infer.parameters(), lr=lr)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0X8z7swjtGGN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "d69a2043-138d-4a5b-c7b3-dd5807b530f8"
      },
      "source": [
        "model_infer"
      ],
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "infer(\n",
              "  (w_embeddings): Embedding(13807, 300)\n",
              "  (linear1): Linear(in_features=300, out_features=50, bias=True)\n",
              "  (linear2): Linear(in_features=50, out_features=13807, bias=True)\n",
              "  (d_embeddings): Embedding(1, 300)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tp5ARx04l5BH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "57667062-caf1-4001-a27c-9c34ba4cd47d"
      },
      "source": [
        "if(is_cuda):\n",
        "  model_infer.cuda()\n",
        "count=0\n",
        "for j in range(300):\n",
        "  # train_loss=[]\n",
        "  for i,(x,y) in enumerate(infer_loader):\n",
        "    count+=1\n",
        "    model_infer.zero_grad()\n",
        "    if(is_cuda):\n",
        "      x, y = x.cuda(), y.cuda()\n",
        "    out=model_infer(x)\n",
        "\n",
        "    loss=criterion(out,y)\n",
        "    loss.backward()\n",
        "\n",
        "    # train_loss.append(loss.item())\n",
        "    \n",
        "    optimizer.step()\n",
        "    # if i%400==0:\n",
        "    #   print('epoch {},batch={},running train loss={}'.format(j+1,i,np.average(train_loss))) \n",
        "  if j==0:\n",
        "    print(\"epoch {} , loss= {}\".format(j+1,loss.item())) \n",
        "\n",
        "  if (j+1)%100==0:\n",
        "    print(\"epoch {} , loss= {}\".format(j+1,loss.item())) "
      ],
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1 , loss= 14.112100601196289\n",
            "epoch 100 , loss= 8.480064392089844\n",
            "epoch 200 , loss= 6.30994987487793\n",
            "epoch 300 , loss= 5.495576858520508\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "inzbz-uvm4NQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "8c650fd4-7d24-431a-ec63-a566827783e6"
      },
      "source": [
        "model_infer.linear1.weight"
      ],
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Parameter containing:\n",
              "tensor([[ 0.2298,  0.1238,  0.3842,  ...,  0.1489,  0.1802,  0.0185],\n",
              "        [-0.1405,  0.0601,  0.0624,  ...,  0.1533, -0.1157, -0.0396],\n",
              "        [ 0.3770, -0.4081, -0.3381,  ...,  0.4333,  0.0632, -0.2612],\n",
              "        ...,\n",
              "        [-0.2157,  0.0762,  0.1830,  ..., -0.1227, -0.2109, -0.3166],\n",
              "        [-0.2073,  0.2350,  0.1430,  ..., -0.0952, -0.2026, -0.0797],\n",
              "        [-0.0725, -0.0123,  0.0772,  ...,  0.0566,  0.0515,  0.4032]],\n",
              "       device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LGqkqQ7wXsVY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_infer.state_dict()['d_embeddings.weight']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IN9pxTQYYO2y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.state_dict()['d_embeddings.weight'][0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vATbJjJVaFU6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}