ass1_ip.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import csv\n",
    "\n",
    "\"\"\" given a file path returns a list of tokens\"\"\"\n",
    "def clean(file_path):\n",
    "    opn = open(file_path.format(0)).read()\n",
    "    opn = opn.replace(\"\\n\",\" \")\n",
    "    opn = opn.replace(\"’ \",\"’\")\n",
    "    return opn.split(\" \")\n",
    "\n",
    "\"\"\" given a file path returns a list of paragraphs\"\"\"\n",
    "def sep_paras(file_path):\n",
    "    lst = []\n",
    "    opn = open(file_path.format(0)).read()\n",
    "    paralist = opn.split(\"\\n\")\n",
    "    for para in paralist:\n",
    "        tknzd = para.replace(\"’ \",\"’\")\n",
    "        tknzd = para.split(\" \")\n",
    "        lst.append(tknzd)\n",
    "    return lst\n",
    "\n",
    "\"\"\" sets all files to be used throughout code\"\"\"\n",
    "testtext = clean('Assignment1_resources/test/test.txt')\n",
    "obamatrain = clean('Assignment1_resources/train/obama.txt')\n",
    "trumptrain = clean('Assignment1_resources/train/trump.txt')\n",
    "obamadev = clean('Assignment1_resources/development/obama.txt')\n",
    "trumpdev = clean('Assignment1_resources/development/trump.txt')\n",
    "paras = sep_paras('Assignment1_resources/test/test.txt')\n",
    "\n",
    "chop = testtext[0:250]\n",
    "\n",
    "\"\"\" receives list of tokens and returns dictionary with unique tokens\n",
    "    as keys and unigram frequency of tokens throughout the text as values\"\"\"\n",
    "def getUniCount(txt):\n",
    "    countdict = dict()\n",
    "    for word in txt:\n",
    "        if word in countdict: \n",
    "            countdict[word] += 1\n",
    "        else:\n",
    "            countdict[word] = 1\n",
    "    return countdict\n",
    "\n",
    "\"\"\" receives list of tokens and returns dictionary with unique tokens\n",
    "    as keys and unigram probabilities of tokens occuring in whole doc as values\"\"\"\n",
    "def getUniProb(count_dict):\n",
    "    probdict = dict()\n",
    "    for k,v in count_dict.items():\n",
    "        probdict[k] = v/len(testtext)\n",
    "    return probdict\n",
    "\n",
    "\"\"\" receives list of tokens and returns a 2D dictionary for bigram frequencies\"\"\"\n",
    "def getBiCount(txt):\n",
    "    countdict = dict()\n",
    "    for i in range(len(txt)-1):\n",
    "        curr = txt[i]\n",
    "        nxt = txt[i+1]\n",
    "        if curr in countdict.keys():\n",
    "            if nxt in countdict[curr]:\n",
    "                countdict[curr][nxt] += 1\n",
    "            else:\n",
    "                countdict[curr][nxt] = 1\n",
    "        else:\n",
    "            countdict[curr] = {nxt: 1}\n",
    "    return countdict\n",
    "\n",
    "\"\"\" receives list of tokens and returns a 2D dictionary for bigram probabilities\"\"\"\n",
    "def getBiProb(unicount_dict,bicount_dict):\n",
    "    probdict = dict()\n",
    "    for k1,v1 in bicount_dict.items():\n",
    "        probdict[k1] = dict()\n",
    "        for k2,v2 in v1.items():\n",
    "            probdict[k1][k2] = v2 / unicount_dict[k1]\n",
    "    return probdict\n",
    "\n",
    "\"\"\" returns an int indicating the (rounded) average length of all\n",
    "    sentences in the corpus.\"\"\"\n",
    "def ave_sentence_length(txt):\n",
    "    punctuation = filter(lambda x: x[1] in ['.', '?', '!'], enumerate(txt))\n",
    "    indices = [i[0] for i in punctuation]\n",
    "    subtract = [0] + indices[:-1]\n",
    "    difference = np.subtract(indices, subtract)\n",
    "    avg = np.mean(difference)\n",
    "    return int(np.rint(avg))\n",
    "\n",
    "\"\"\" receives list of float values and returns new list of float \n",
    "    values summing to 1. fixes numpy random.choice glitch to ensure\n",
    "    weighted probabilities sum to 1\"\"\"\n",
    "def div_sum(vals):\n",
    "    newlst = []\n",
    "    sv = sum(vals)\n",
    "    for v in vals:\n",
    "        newlst.append(v / sv)\n",
    "    return newlst\n",
    "\n",
    "\"\"\" Returns a randomly generated sentence made up of unigrams.\n",
    "        The sentence terminates when either:\n",
    "                1. The next unigram to add is an ending punctuation\n",
    "                   term: '.', '!', or '?'\n",
    "                2. The sentence is as long as the average sentence\n",
    "                   of the corpus \"\"\"\n",
    "def unigram_sentence(txt,seed):\n",
    "    if seed != \"\": sentence = ['<s>'] + seed.split(\" \")\n",
    "    else: sentence = ['<s>']\n",
    "    unigrams = getUniCount(txt)\n",
    "    while len(sentence) <= ave_sentence_length(txt):\n",
    "        unigram_val = np.random.choice(list(getUniProb(getUniCount(txt)).keys()), \n",
    "                                       p = div_sum(list(getUniProb(getUniCount(txt)).values())))\n",
    "        sentence.append(unigram_val)\n",
    "        if (unigram_val in ['.', '!', '?']):\n",
    "            return sentence + ['</s>']\n",
    "    return sentence + ['</s>']\n",
    "\n",
    "\"\"\" Returns a randomly generated sentence made up of bigrams.\n",
    "    The sentence terminates when either:\n",
    "            1. The next unigram to add is an ending punctuation\n",
    "               term: '.', '!', or '?'\n",
    "            2. The sentence is as long as the average sentence of the\n",
    "               corpus \"\"\"\n",
    "def bigram_sentence(txt,seed):\n",
    "        if seed != \"\": \n",
    "            sentence = ['<s>'] + seed.split(\" \")\n",
    "            bigram_start = sentence[-1]\n",
    "        else:\n",
    "            bigram_start = np.random.choice(list(getUniProb(getUniCount(txt)).keys()), \n",
    "                                            p = div_sum(list(getUniProb(getUniCount(txt)).values())))\n",
    "            sentence = ['<s>', bigram_start] \n",
    "        \n",
    "        # Generate sentences\n",
    "        while bigram_start not in ['.', '!', '?'] and len(sentence) <= ave_sentence_length(txt):\n",
    "            start_dict = getBiProb(getUniCount(txt),getBiCount(txt))[bigram_start]\n",
    "            next_word = np.random.choice(list(start_dict.keys()), p = div_sum(list(start_dict.values())))\n",
    "            sentence.append(next_word)\n",
    "            bigram_start = next_word\n",
    "            \n",
    "        return sentence + ['</s>'] \n",
    "\n",
    "\"\"\" receives a unigram count dictionary and returns a new \n",
    "    dictionary with all tokens with only one count substituted to a <unk> tag\"\"\"\n",
    "def uniCountUnk(old_dict):\n",
    "    new_dict = {}\n",
    "    new_dict[\"<unk>\"] = 0\n",
    "    for k,v in old_dict.items():\n",
    "        if v == 1:\n",
    "            new_dict.pop(k, None)\n",
    "            new_dict[\"<unk>\"] += 1 \n",
    "        else:\n",
    "            new_dict[k] = v\n",
    "    return new_dict\n",
    "\n",
    "\"\"\" receives a token list and returns a new 2D bicount dictionary\n",
    "    with all one-time occuring tokens substituted to a <unk> tag\"\"\"\n",
    "def biCountUnk(txt):\n",
    "    newlst = []\n",
    "    newlst = txt\n",
    "    unicount_old = getUniCount(txt)\n",
    "    for k,v in unicount_old.items():\n",
    "        if v == 1:\n",
    "            newlst[newlst.index(k)] = \"<unk>\"\n",
    "    return getBiCount(newlst)\n",
    "\n",
    "\"\"\" SMOOTHED BIGRAM HELPERS\n",
    "    first chunk of the formula\"\"\"\n",
    "def getMax(first,second,bicount_dict):\n",
    "    if second not in bicount_dict[first] or bicount_dict[first][second] - 0.75 < 0:\n",
    "        return 0\n",
    "    else:\n",
    "        return (bicount_dict[first][second] - 0.75) / sum(bicount_dict[first].values())\n",
    "    \n",
    "\"\"\" returns lambda value for given first word\"\"\"\n",
    "def getLambda(first,bicount_dict):\n",
    "    return 0.75 / sum(bicount_dict[first].values()) * len(bicount_dict[first])\n",
    "\n",
    "\"\"\" returns pkn value for given second word\"\"\"\n",
    "def getPKN(second,bicount_dict):\n",
    "    counter = 0\n",
    "    total = 0\n",
    "    for k,v in bicount_dict.items():\n",
    "        if second in v:\n",
    "            counter += 1\n",
    "        for k2,v2 in v.items():\n",
    "            total += 1\n",
    "    return (counter / total)\n",
    "\n",
    "\"\"\" returns a lambda dictionary where the keys are all possible words in the text \n",
    "    and each key is mapped to its lambda value\"\"\"\n",
    "def makeLambdaDict(bicount_dict):\n",
    "    probdict = dict()\n",
    "    for k in bicount_dict.keys():\n",
    "        probdict[k] = getLambda(k,bicount_dict)\n",
    "    return probdict\n",
    "   \n",
    "\"\"\" returns a pkn dictionary where the keys are all possible words in the text \n",
    "    and each key is mapped to its pkn value\"\"\"\n",
    "def makePKNDict(bicount_dict):\n",
    "    probdict = dict()\n",
    "    for k in bicount_dict.keys():\n",
    "        probdict[k] = getPKN(k,bicount_dict)\n",
    "    return probdict\n",
    "\n",
    "\"\"\" uses the Kneser Ney method to calculate bigram probabilities. \n",
    "    returns a 2D dictionary with all possible pairs of words\"\"\"\n",
    "def smoothed_bigram(lambda_dict,pkn_dict,bicount_dict):\n",
    "    probdict = dict()\n",
    "    for w1 in bicount_dict.keys():\n",
    "        probdict[w1] = dict()\n",
    "        for w2 in bicount_dict.keys():\n",
    "            probdict[w1][w2] = getMax(w1,w2,bicount_dict) + lambda_dict[w1] * pkn_dict[w2]\n",
    "    return probdict\n",
    "\n",
    "\"\"\" computes maximum entropy of the given development set\"\"\"\n",
    "def perplexity(dev_txt,bicount_dict):\n",
    "    new_sum = 0\n",
    "    for i in range(len(dev_txt)-1):\n",
    "        w1 = dev_txt[i]\n",
    "        w2 = dev_txt[i+1]\n",
    "        if w1 in bicount_dict.keys():\n",
    "            if w2 in bicount_dict[w1].keys(): \n",
    "                x = bicount_dict[w1][w2]\n",
    "            else:\n",
    "                x = bicount_dict[w1][\"<unk>\"]\n",
    "        else:\n",
    "            if w2 in bicount_dict[\"<unk>\"].keys(): \n",
    "                x = bicount_dict[\"<unk>\"][w2]\n",
    "            else:\n",
    "                x = bicount_dict[\"<unk>\"] [\"<unk>\"] \n",
    "        new_sum += -1 * np.log(x) # where x is the probability of bigram\n",
    "    result = np.exp((1.0/len(dev_txt))*new_sum)\n",
    "    return result # for all bigrams in N where N is len of dev_corpus\n",
    "\n",
    "def classification(trump_perp,obama_perp,trump_smoo,obama_smoo):\n",
    "    lst = []\n",
    "    for para in paras:\n",
    "        trump_delta = abs(perplexity(para,trump_smoo) - trump_perp)\n",
    "        obama_delta = abs(perplexity(para,obama_smoo) - obama_perp)\n",
    "        if trump_delta < obama_delta:\n",
    "            lst.append(1)\n",
    "        else:\n",
    "            lst.append(0)\n",
    "    return lst\n",
    "\n",
    "def makecsv(lst):\n",
    "    with open(\"classification.csv\", 'w', newline='') as myfile:\n",
    "        wr = csv.writer(myfile, delimiter=',', quoting=csv.QUOTE_ALL)\n",
    "        wr.writerow(['Id', 'Prediction'])\n",
    "        for i in range(len(lst) - 1):\n",
    "            wr.writerow([i, lst[i]])\n",
    "            \n",
    "    \n",
    "\n",
    "# pre-set arguments for smoothed_bigram function\n",
    "bicount_obama = biCountUnk(obamatrain)\n",
    "bicount_trump = biCountUnk(trumptrain)\n",
    "\n",
    "lambda_d_trump = makeLambdaDict(bicount_trump)\n",
    "lambda_d_obama = makeLambdaDict(bicount_obama)\n",
    "\n",
    "pkn_d_trump = makePKNDict(bicount_trump)\n",
    "pkn_d_obama = makePKNDict(bicount_obama)\n",
    "\n",
    "# create smoothed_bigram dictionary\n",
    "smoothed_trump = smoothed_bigram(lambda_d_trump,pkn_d_trump,bicount_trump)\n",
    "smoothed_obama = smoothed_bigram(lambda_d_obama,pkn_d_obama,bicount_obama)\n",
    "\n",
    "# set perplexity variables\n",
    "trump_perp = perplexity(trumptrain,smoothed_trump)\n",
    "obama_perp = perplexity(obamatrain,smoothed_obama)\n",
    "\n",
    "#print(trump_perp)\n",
    "#print(obama_perp)\n",
    "\n",
    "class_lst = classification(trump_perp,obama_perp,smoothed_trump,smoothed_obama)\n",
    "print(class_lst)\n",
    "makecsv(class_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}