tf_idf_similarity.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tf_idf_similarity.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPjS9szX4yKPhxr0/tbmCeH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shashank-m/covid_mining_papers/blob/master/tf_idf_similarity.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bi0Zrl_y6gmD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.cluster import KMeans \n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "import nltk\n",
        "from nltk import word_tokenize, sent_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "import re\n",
        "import inflect\n",
        "import time\n",
        "from sklearn.preprocessing import normalize\n",
        "from nltk.corpus import stopwords \n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "p = inflect. engine()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NBpe70rR6uGI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wg0TPove6uN8",
        "colab_type": "code",
        "outputId": "d0c7be95-3be5-4353-ab84-8d2344d0d6f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "drive.mount('/content/drive/')\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7w6PFXlc6uPR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "import os\n",
        "\n",
        "filenames=os.listdir(\"/content/drive/My Drive/covid_nlp/2020-03-13/biorxiv_medrxiv/biorxiv_medrxiv\")\n",
        "biorxiv=\"/content/drive/My Drive/covid_nlp/2020-03-13/biorxiv_medrxiv/biorxiv_medrxiv/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FvodmhVi6uQl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_files = []\n",
        "\n",
        "for filename in filenames:\n",
        "    filename = biorxiv + filename\n",
        "    file = json.load(open(filename, 'rb'))\n",
        "    all_files.append(file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VAmePox06uR2",
        "colab_type": "code",
        "outputId": "cfd6b0a2-9fa1-44d0-9625-e3e5acf003c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "stop_words = set(stopwords.words('english')) "
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5_YgZfzS6uTK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def remove_punct(s):\n",
        "  s=re.sub(r'[!.,;:-]',' ',s) \n",
        "  s=' '.join(s.split())\n",
        "  return s\n",
        "def replace_nums(s):\n",
        "  numbers_in_string=re.findall(r'[0-9]+',s) # finds all numbers in string  \n",
        "  for number in numbers_in_string:\n",
        "    x=p.number_to_words(int(number)) # convert number to word.\n",
        "    s=re.sub(number,x,s)  \n",
        "  return s\n",
        "def tokenise_lower(s):\n",
        "  tokens=nltk.word_tokenize(s)\n",
        "  word_tokens= [i.lower() for i in tokens]\n",
        "  return [w for w in word_tokens if not w in stop_words] # removes stop words.\n",
        " \n",
        "def unique_words_entire_data(data):\n",
        "  unique=set()\n",
        "  for i in data.iteritems():\n",
        "    unique=unique.union(set(i[1]))\n",
        "  return len(unique)  \n",
        "def convert_list_string(l):\n",
        "  return ' '.join(l)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TM785-F-6uUe",
        "colab_type": "code",
        "outputId": "d9dbeb28-6791-4378-e8c3-89cae0cb30e3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "titles_in_biorxiv=[]\n",
        "for f in all_files:\n",
        "  t=f['metadata']['title']\n",
        "  titles_in_biorxiv.append(t)\n",
        "\n",
        "titles_in_biorxiv=pd.Series(titles_in_biorxiv)\n",
        "\n",
        "titles_in_biorxiv=titles_in_biorxiv.apply(remove_punct).apply(replace_nums).apply(tokenise_lower)\n",
        "print(\"No of unique words in titles of biorxiv dataset = {}\".format(unique_words_entire_data(titles_in_biorxiv)))\n",
        "\n",
        "# tfidf takes inpit as string. Not list. So convert list to string below.\n",
        "titles_in_biorxiv=titles_in_biorxiv.apply(convert_list_string)\n",
        "\n",
        "tf_idf_vec=TfidfVectorizer(max_features=3000)\n",
        "\n",
        "tf_idf=tf_idf_vec.fit_transform(titles_in_biorxiv)\n",
        "\n",
        "tf_idf_norm = normalize(tf_idf)\n",
        "\n",
        "tf_idf_array = tf_idf_norm.toarray()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No of unique words in titles of biorxiv dataset = 2876\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wYRqYVDE7HTs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def top_k_results(query,tf_idf_vec,tf_idf_array,k):\n",
        "  \"\"\"\n",
        "  compute tfidf vector of query.\n",
        "  find similarity with each row od tf_array.\n",
        "  sort by highest k similarity scores and return these documents.\n",
        "  \"\"\"\n",
        "  query=query.apply(remove_punct).apply(replace_nums).apply(tokenise_lower).apply(convert_list_string)\n",
        "\n",
        "  tf_idf_query=normalize(tf_idf_vec.transform(query)).toarray()\n",
        "\n",
        "  cos_sim=cosine_similarity(tf_idf_query,tf_idf_array)\n",
        "  \n",
        "  top_k_documents=np.flip(np.argsort(cos_sim,axis=1),axis=1)[:,:k]\n",
        "\n",
        "  \"\"\"\n",
        "   each row of top_k_documents correspond to each query.\n",
        "   They contain the indices of the documents.\n",
        "  \"\"\"\n",
        "  return top_k_documents.tolist()\n",
        "k=3  \n",
        "query=pd.Series(['Efforts targeted at a universal coronavirus vaccine','Therapeutics research'])\n",
        "top_k_documents=top_k_results(query,tf_idf_vec,tf_idf_array,k)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bsZB1I9J7HZM",
        "colab_type": "code",
        "outputId": "5d2c998f-9b21-4c2c-af2e-3dbf9d0c8dbf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "for i,q in enumerate(query):\n",
        "  print('For query \"{}\"'.format(q))\n",
        "  print('The top {} documents are :'.format(k))\n",
        "  print('\\n')\n",
        "  for title in titles_in_biorxiv[top_k_documents[i]]:\n",
        "    print(title)\n",
        "  print('*****************') \n",
        "  print('\\n') "
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "For query \"Efforts targeted at a universal coronavirus vaccine\"\n",
            "The top 3 documents are :\n",
            "\n",
            "\n",
            "simple model assess wuhan lock effect region efforts covid nineteen epidemic china mainland\n",
            "ai aided design novel targeted covalent inhibitors sars cov two\n",
            "den im dengue virus identification shotgun targeted metagenomics\n",
            "*****************\n",
            "\n",
            "\n",
            "For query \"Therapeutics research\"\n",
            "The top 3 documents are :\n",
            "\n",
            "\n",
            "efficacy revealed single cell analysis antiviral therapeutics\n",
            "two systematic review evaluation zika virus forecasting prediction research three public health emergency international concern four five kobres p\n",
            "article type research article title radar dengue virus infections natural populations aedes aegypti mosquitoes running title dengue virus maintenance mosquito vectors\n",
            "*****************\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OWqUlyy29siD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}