Step_1_data_processing_TF-IDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a variable word_data as input.  This is a list of text documents. Then tokenize, stem and TF-IDF vecorize.  Output should be a set of TF-IDF vectors for each DOI.\n",
    "\n",
    "This all takes place in-memory.  It may be that this needs edited if we want to grow the dataset to a larger size.  For now, it should give better speed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------\n",
      "Step 1:  Calculating TF-IDF data\n",
      "2018-02-21 21:10:11.838369\n",
      "------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "print('------------------------------------------------------')\n",
    "print('Step 1:  Calculating TF-IDF data')\n",
    "from datetime import datetime as dt\n",
    "t_start = dt.now()\n",
    "print(t_start)\n",
    "print('------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, TfidfTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.pipeline import Pipeline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import configuration\n",
    "Using a separate file config.py to create shared variables for the various notebooks in this project."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from config import Config as c\n",
    "\n",
    "# inputs\n",
    "\n",
    "years = c.years\n",
    "\n",
    "\n",
    "word_datapath = c.word_datapath\n",
    "tfidf_datapath = c.tfidf_datapath\n",
    "labels_path = c.labels_path\n",
    "\n",
    "vectorizer_datapath = c.vectorizer_datapath\n",
    "filepaths_pkl = c.filepaths_pkl\n",
    "\n",
    "# outputs\n",
    "dois_pkl = c.dois_pkl\n",
    "all_dois_pkl = c.all_dois_pkl\n",
    "working_data = c.working_data\n",
    "vocab_p = c.vocab_p\n",
    "idf_p = c.idf_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# load word data and corresponding doi_ls\n",
    "with open(word_datapath, 'rb') as f:\n",
    "    word_data = pickle.load(f)\n",
    "with open(dois_pkl, 'rb') as f:\n",
    "    dois = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the template dataframe for that will eventually take the data for visualisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18355, 27)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import pandas as pd\n",
    "# import numpy as np\n",
    "data = pd.read_csv(working_data, index_col=0)\n",
    "np.shape(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# DEFINE DOI_LS VARIABLE HERE\n",
    "dois = list(data['DI'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18355, 18355)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dois), len(word_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataFrame initialised\n"
     ]
    }
   ],
   "source": [
    "# write to file\n",
    "data.to_csv(working_data)\n",
    "print('DataFrame initialised')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate TF-IDF data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting TF-IDF indexing.\n"
     ]
    }
   ],
   "source": [
    "print('Starting TF-IDF indexing.')\n",
    "# stopwords\n",
    "from sklearn.feature_extraction import text\n",
    "\n",
    "my_words = ['et','al','use','article','introduction','abstract','title', 'nan'] # note that there are 'NaN's in WoS data!\n",
    "\n",
    "# Add custom stopwords here.  E.g. ['sensor','network','data','node']  are so common in DSN\n",
    "# that they appear in almost every paper and make it hard to differentiate between clusters.  \n",
    "custom_stops = []\n",
    "my_words = my_words+custom_stops\n",
    "\n",
    "my_stop_words = text.ENGLISH_STOP_WORDS.union(my_words)\n",
    "my_stop_words = set(my_stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "from tools import strip_stem\n",
    "word_data = [strip_stem(doc) for doc in word_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of TF-IDF matrix: (18355, 5000)\n",
      "I.e. 18355 documents with 5000 unique words in them\n"
     ]
    }
   ],
   "source": [
    "# tf-idf vectorizer\n",
    "vectorizer = TfidfVectorizer(stop_words=my_stop_words,\n",
    "#                              max_df = 0.99, # ignore the most frequent 1% of words\n",
    "#                             min_df = 2, # ignore words that appear in this many docs or fewer\n",
    "#                             ngram_range =(1,2),# causes slow-down\n",
    "                            use_idf = True,\n",
    "                            max_features = 5000) # save some memory and reduce dimensionality.  \n",
    "tfidf = vectorizer.fit_transform(word_data)\n",
    "vocab = vectorizer.vocabulary_\n",
    "idf = vectorizer.idf_\n",
    "shp = np.shape(tfidf)\n",
    "print('Shape of TF-IDF matrix:', shp)\n",
    "print('I.e. {} documents with {} unique words indexed in them'.format(shp[0],shp[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF-IDF indexing complete. Writing to file.\n"
     ]
    }
   ],
   "source": [
    "# cosine distances\n",
    "# sim = tfidf*tfidf.T\n",
    "# cosine_sims = sim.todense() # this can be memory-hungry and isn't needed for the visualisation.\n",
    "print('TF-IDF indexing complete. Writing to file.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing data to file\n"
     ]
    }
   ],
   "source": [
    "print('Writing data to file')\n",
    "pickle.dump(tfidf, open(tfidf_datapath,'wb'))\n",
    "pickle.dump(vectorizer, open(vectorizer_datapath,'wb'))\n",
    "pickle.dump(vocab, open(vocab_p,'wb'))\n",
    "pickle.dump(idf, open(idf_p,'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done at  2018-02-21 21:18:20.592957 in  0:08:08.754588\n"
     ]
    }
   ],
   "source": [
    "t =dt.now()\n",
    "print('Done at ',t, 'in ',t-t_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}