DataPreprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "from IPython.display import display, Math, Latex, Image\n",
    "from PIL import Image\n",
    "import random\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate vocabulary and char_to_idx, idx_to_char dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "502\n",
      "{'\\\\perp': 324, '\\\\leq': 260, '\\\\setminus': 375, '\\\\supset': 404, '\\\\raisebox': 342, '0': 14, '\\\\Re': 107, '\\\\smallint': 385, '\\\\surd': 406, '\\\\hspace': 220, '\\\\label': 235, '@': 38, '\\\\rlap': 367, 'J': 48, '\\\\rightharpoonup': 364, 'P': 55, '\\\\textbf': 408, '\\\\overbrace': 317, '\\\\Sigma': 111, '\\\\scriptstyle': 372, '\\\\rho': 351, '\\\\tilde': 419, '`': 463, '--': 10, '\\\\mathbin': 276, 'p': 482, '\\\\asymp': 128, '\\\\Psi': 106, '\\\\Omega': 102, '\\\\nabla': 293, '+': 7, '\\\\Longrightarrow': 100, '\\\\mathit': 278, '\\\\emptyset': 191, ';': 33, '\\\\colon': 169, '\\\\bigcup': 138, 'K': 49, '\\\\protect': 335, '\\\\leftarrow': 256, '[': 66, '\\\\prec': 331, '\\\\otimes': 316, '\\\\bigoplus': 144, '\\\\aleph': 120, '\\\\cdots': 163, '\\\\cdotp': 162, 'k': 476, '\\\\nonumber': 300, '\\\\cong': 170, '\\\\displaystyle': 183, '\\\\end{array}': 192, '{': 495, '~': 498, '\\\\begin{array}': 133, '\\\\equiv': 197, '\\\\widetilde': 453, '\\\\ll': 264, '\\\\Biggr': 84, '\\\\right.': 353, '\\\\le': 244, '\\\\star': 397, '\\\\neq': 296, '\\\\lq': 271, '\\\\alpha': 121, '#END': 501, '\\\\pmod': 329, '&': 2, '\\\\raise': 341, '6': 27, '\\\\amalg': 122, '\\\\lambda': 236, 'F': 44, '\\\\vspace': 449, '\\\\tiny': 421, 'V': 61, 'f': 470, 'mm': 479, '\\\\smallskip': 386, '\\\\ddots': 179, '\\\\kappa': 232, 'v': 490, '\\\\sigma': 378, '\\\\Rightarrow': 108, 'ule': 489, '\\\\vert': 444, '\\\\cup': 172, '\\\\kern': 233, '0.1': 15, '0.3': 17, '0.5': 19, '0.4': 18, '\\\\Theta': 112, '\\\\wp': 454, '\\\\varrho': 436, '!': 0, '\\\\textsf': 412, '1': 20, '\\\\arraycolsep': 124, '\\\\sqcap': 392, '\\\\smash': 387, 'A': 39, '\\\\Vert': 114, '\\\\hline': 216, '\\\\beta': 134, '\\\\scriptsize': 371, '\\\\longleftarrow': 266, 'Q': 56, 'a': 464, '\\\\grave': 211, '\\\\texttt': 414, '\\\\bmod': 152, '\\\\hrule': 219, 'q': 484, '\\\\lefteqn': 257, '\\\\em': 190, '\\\\geq': 209, '\\\\crcr': 171, '\\\\vskip': 448, '\\\\longmapsto': 268, '\\\\makebox': 273, '\\\\rbrack': 345, '\\\\left|': 259, '\\\\medskip': 284, ',': 8, '\\\\hfil': 214, '\\\\left[': 248, '<': 34, '\\\\left(': 245, 'L': 50, '\\\\ddag': 176, '\\\\': 68, '\\\\left<': 247, '\\\\buildrel': 156, 'l': 477, '\\\\framebox': 206, '\\\\special': 391, 'h': 472, '\\\\AA': 79, '\\\\prod': 333, '\\\\right|': 366, '\\\\lbrace': 240, '\\\\lbrack': 241, '\\\\right\\\\rbrace': 356, '\\\\left\\\\vert': 253, '\\\\sb': 368, '\\\\sc': 369, '\\\\bot': 154, '\\\\sl': 382, '\\\\right\\\\rbrack': 357, '\\\\noalign': 298, '\\\\right]': 362, \"'\": 3, '\\\\arraystretch': 125, '\\\\theta': 416, '\\\\leftrightarrow': 258, '\\\\psi': 337, '\\\\not': 302, '7': 28, '\\\\protectu': 336, '\\\\lower': 270, '\\\\boldmath': 153, '\\\\SS': 110, 'G': 45, '\\\\right>': 354, '\\\\ldots': 243, '\\\\right)': 352, '\\\\bullet': 157, 'W': 62, '\\\\parbox': 322, '\\\\varphi': 434, '\\\\textup': 415, '\\\\enskip': 193, '\\\\varsigma': 437, 'g': 471, '\\\\llap': 265, 'w': 491, '\\\\breve': 155, '\\\\ae': 119, '\\\\underbrace': 428, '\\\\textnormal': 410, '\\\\Delta': 88, '\\\\dot': 185, 'Y': 64, '\\\\oplus': 314, '\\\\mathcal': 277, '\"': 1, '\\\\circ': 166, '\\\\mathversion': 283, 't': 487, '2': 21, '\\\\upsilon': 432, 'B': 40, '\\\\fboxsep': 201, '\\\\chi': 165, '\\\\to': 422, '\\\\doteq': 186, 'R': 57, '\\\\tt': 426, '\\\\flat': 202, '\\\\lgroup': 262, 'b': 465, '\\\\qquad': 339, '\\\\rfloor': 349, 'r': 485, '\\\\enspace': 194, '\\\\Phi': 104, '\\\\epsilon': 196, '\\\\space': 390, '\\\\varpi': 435, '\\\\bigcap': 137, '\\\\operatorname*': 313, '\\\\Pi': 105, '-': 9, '\\\\small': 384, '\\\\dag': 174, '\\\\ast': 127, '\\\\mapsto': 274, '\\\\thinspace': 418, '=': 35, '\\\\bigotimes': 145, '\\\\oint': 308, '\\\\left\\\\lbrace': 250, '\\\\imath': 223, '\\\\diamond': 181, '\\\\left\\\\lbrack': 251, 'M': 51, '\\\\smile': 388, '\\\\overline': 319, '\\\\ell': 189, '\\\\Large': 96, '\\\\mp': 288, '\\\\top': 423, '\\\\ddagger': 177, ']': 460, '\\\\mu': 290, '#UNK': 499, '\\\\ddot': 178, 'm': 478, '\\\\rangle': 343, '\\\\vrule': 447, '\\\\vline': 445, '\\\\normalsize': 301, '\\\\right\\\\|': 360, '\\\\right\\\\}': 361, '\\\\bigtriangledown': 148, '\\\\oslash': 315, '\\\\bigsqcup': 147, '(': 4, '\\\\cdot': 161, '8': 29, '\\\\lfloor': 261, '\\\\bar': 132, '\\\\pm': 328, '\\\\left\\\\lfloor': 252, '\\\\pi': 327, '\\\\supseteq': 405, 'H': 46, 'X': 63, '\\\\multiput': 292, '\\\\Gamma': 90, '\\\\fbox': 200, '\\\\textrm': 411, '\\\\Huge': 91, '\\\\tau': 407, '\\\\mskip': 289, 'x': 492, '\\\\mathrm': 281, '|': 496, '\\\\odot': 307, 'j': 475, '\\\\bf': 135, '\\\\pounds': 330, '\\\\hphantom': 218, '\\\\overrightarrow': 320, '\\\\propto': 334, '\\\\sqcup': 393, '\\\\textstyle': 413, '\\\\line': 263, '\\\\triangleright': 425, '\\\\triangle': 424, '3': 22, '\\\\bigwedge': 150, '\\\\hfill': 215, '\\\\omega': 309, '\\\\land': 237, '\\\\it': 228, 'C': 41, '\\\\hookrightarrow': 217, '\\\\sharp': 377, '\\\\slash': 383, '\\\\uparrow': 431, 'S': 58, '\\\\in': 224, 'c': 466, 'z': 494, '\\\\overleftarrow': 318, 's': 486, '\\\\varepsilon': 433, '\\\\large': 239, '\\\\ensuremath': 195, '\\\\textit': 409, 'cm': 467, '\\\\frac': 205, '\\\\hat': 212, '\\\\eta': 198, '\\\\forall': 204, '\\\\times': 420, '8.5': 30, 'pt': 483, '\\\\dots': 187, '.': 12, '\\\\mathop': 279, '\\\\put': 338, '\\\\atop': 129, '\\\\longleftrightarrow': 267, '\\\\Bigr': 87, 'N': 52, '\\\\Bigg': 81, '0.14': 16, '\\\\wedge': 451, '\\\\Bigm': 86, '\\\\Bigl': 85, '^': 461, '\\\\jmath': 231, 'n': 480, '\\\\footnotesize': 203, '\\\\diamondsuit': 182, '\\\\vcenter': 439, '\\\\ni': 297, '\\\\acute': 118, '\\\\ne': 295, '\\\\rightarrow': 363, '\\\\Xi': 115, '#START': 500, '\\\\iota': 227, '\\\\stackrel': 396, '\\\\Im': 92, 'in': 474, '\\\\nu': 304, 'Object]': 54, '\\\\S': 109, '\\\\rbrace': 344, '\\\\phi': 326, '\\\\hss': 221, ')': 5, '\\\\backslash': 131, '\\\\#': 70, '\\\\langle': 238, '\\\\multicolumn': 291, '9': 31, '\\\\vdash': 440, '\\\\unitlength': 430, '\\\\check': 164, 'I': 47, '\\\\notin': 303, '[object': 67, '\\\\ominus': 310, '\\\\left.': 246, '\\\\cite': 168, '\\\\operatorname': 312, '\\\\hbar': 213, '\\\\mathrel': 280, '\\\\null': 305, 'i': 473, 'y': 493, '\\\\big': 136, '\\\\sqrt': 394, '\\\\succ': 401, '\\\\itshape': 229, '\\\\rgroup': 350, '\\\\right\\\\rfloor': 358, '\\\\infty': 225, '\\\\ref': 346, '\\\\:': 77, '\\\\;': 78, '\\\\xi': 455, '\\\\&': 71, \"\\\\'\": 72, '\\\\mathsf': 282, '4': 25, '\\\\!': 69, '\\\\/': 76, '\\\\,': 74, '\\\\-': 75, '\\\\*': 73, 'D': 42, '\\\\P': 103, '\\\\_': 117, '\\\\\\\\': 116, '\\\\longrightarrow': 269, 'T': 59, '\\\\right\\\\vert': 359, '\\\\O': 101, '\\\\L': 93, '\\\\zeta': 456, '\\\\arrowvert': 126, 'd': 468, '\\\\vee': 443, '\\\\|': 458, '\\\\}': 459, '\\\\{': 457, '\\\\vec': 442, '\\\\d': 173, '\\\\b': 130, '\\\\c': 158, '\\\\o': 306, '\\\\l': 234, '\\\\m': 272, '\\\\j': 230, '\\\\Leftrightarrow': 97, '\\\\i': 222, '\\\\strut': 398, '\\\\gamma': 207, '\\\\lceil': 242, '\\\\left\\\\|': 255, '3.2': 24, '3.1': 23, '\\\\subseteq': 400, '\\\\rightleftharpoons': 365, '\\\\Upsilon': 113, '\\\\cal': 159, '\\\\Longleftarrow': 98, '/': 13, '\\\\ooalign': 311, '\\\\cap': 160, '\\\\int': 226, '\\\\sim': 379, '?': 37, '\\\\prime': 332, '\\\\vss': 450, '\\\\setlength': 374, 'O': 53, '_': 462, '\\\\downarrow': 188, '\\\\Lambda': 95, 'o': 481, '\\\\gg': 210, '\\\\ge': 208, '\\\\renewcommand': 348, '\\\\skew': 381, '\\\\setcounter': 373, '\\\\ss': 395, '\\\\delta': 180, '\\\\phantom': 325, '\\\\sp': 389, '*': 6, '\\\\simeq': 380, '\\\\right\\\\rangle': 355, '\\\\vartheta': 438, '\\\\nolinebreak': 299, ':': 32, '\\\\do': 184, '\\\\parallel': 321, '\\\\exists': 199, '\\\\Biggm': 83, '\\\\Biggl': 82, '\\\\LARGE': 94, 'Z': 65, '\\\\approx': 123, '\\\\sf': 376, '\\\\biggr': 141, '\\\\binom': 151, '\\\\relax': 347, '\\\\unboldmath': 427, '\\\\widehat': 452, '\\\\biggl': 140, '}': 497, '\\\\left\\\\langle': 249, '>': 36, '\\\\mathbf': 275, '\\\\dagger': 175, '\\\\left\\\\{': 254, '\\\\scriptscriptstyle': 370, '\\\\partial': 323, '---': 11, '\\\\vphantom': 446, '\\\\Longleftrightarrow': 99, '\\\\mid': 285, '\\\\quad': 340, '5': 26, '\\\\mit': 286, '\\\\mkern': 287, 'E': 43, '\\\\succeq': 402, '\\\\Downarrow': 89, '\\\\natural': 294, 'U': 60, '\\\\vdots': 441, '\\\\underline': 429, '\\\\circle': 167, '\\\\Big': 80, '\\\\bigtriangleup': 149, 'e': 469, '\\\\bigr': 146, '\\\\sum': 403, '\\\\bigg': 139, '\\\\thicklines': 417, 'u': 488, '\\\\bigm': 143, '\\\\bigl': 142, '\\\\subset': 399}\n"
     ]
    }
   ],
   "source": [
    "vocab = open(\"im2markup/latex_vocab.txt\").readlines()\n",
    "formulae = open(\"im2markup/formulas.norm.lst\",'r').readlines()\n",
    "char_to_idx = {x.split('\\n')[0]:i for i,x in enumerate(vocab)}\n",
    "# print len(char_to_idx)\n",
    "char_to_idx['#UNK'] = len(char_to_idx)\n",
    "char_to_idx['#START'] = len(char_to_idx)\n",
    "char_to_idx['#END'] = len(char_to_idx)\n",
    "idx_to_char = {y:x for x,y in char_to_idx.iteritems()}\n",
    "# print char_to_idx['#UNK']\n",
    "# print char_to_idx['#START']\n",
    "# print char_to_idx['#END']\n",
    "print len(char_to_idx)\n",
    "#print(char_to_idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create train, test, validate files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\cline  not found!\n",
      "\\left\\Vert  not found!\n",
      "\\right\\Vert  not found!\n",
      "\\right/  not found!\n",
      "\\bigcirc  not found!\n",
      "\\linethickness  not found!\n",
      "\\emph  not found!\n",
      "\\mathord  not found!\n",
      "\\searrow  not found!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8491/8491 [00:03<00:00, 2516.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num files found in valid set: 5201/8491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "set = \"valid\" # Use train, valid or test to generate corresponding files\n",
    "file_list = open(\"im2markup/\"+set+\".lst\",'r').readlines()\n",
    "set_list = []\n",
    "missing = {}\n",
    "for i,line in enumerate(file_list):\n",
    "    form = formulae[int(line.split()[1])].strip().split()\n",
    "    out_form = [char_to_idx['#START']]\n",
    "    for c in form:\n",
    "        try:\n",
    "            out_form += [char_to_idx[c]]\n",
    "        except:\n",
    "            if c not in missing:\n",
    "                print c, \" not found!\"\n",
    "                missing[c] = 1\n",
    "            else:\n",
    "                missing[c] += 1\n",
    "            out_form += [char_to_idx['#UNK']]\n",
    "    out_form += [char_to_idx['#END']]\n",
    "    set_list.append([line.split()[0],out_form])\n",
    "    \n",
    "buckets = {}\n",
    "import os\n",
    "file_not_found_count = 0\n",
    "for x,y in tqdm(set_list):\n",
    "    if os.path.exists('./images_processed/'+x): \n",
    "        img_shp = Image.open('./images_processed/'+x).size\n",
    "        try:\n",
    "            buckets[img_shp] += [(x,y)]\n",
    "        except:\n",
    "            buckets[img_shp] = [(x,y)]\n",
    "    else:\n",
    "        file_not_found_count += 1\n",
    "\n",
    "print \"Num files found in %s set: %d/%d\"%(set,len(set_list)-file_not_found_count,len(set_list)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'0.9': 1, '\\\\protectE': 1, '\\\\left\\\\lceil': 1, '\\\\rightarrowfill': 1, '\\\\protectZ': 1, '\\\\fill': 1, '0.23': 1, '\\\\brack': 1, '\\\\right[': 1, '20': 1, '\\\\vector': 1, '\\\\protecte': 1, '\\\\protectm': 1, '\\\\mathord': 1, '\\\\atopwithdelims': 1, '\\\\nearrow': 1, '\\\\nulldelimiterspace': 1, '\\\\left]': 1, '\\\\right<': 1, '\\\\^': 1, '\\\\of': 1, '\\\\verb': 1, '\\\\root': 1, '\\\\left>': 1, '\\\\mathstrut': 1, '\\\\right\\\\rceil': 1, '\\\\symbol': 1, '\\\\biggm': 1}\n",
      "Max length of sequence:  152\n",
      "Testing!\n",
      "('35a6b52146.png', [500, 337, 4, 207, 5, 35, 312, 495, 469, 492, 482, 497, 495, 9, 4, 495, 413, 495, 205, 495, 471, 461, 495, 21, 497, 497, 495, 21, 497, 497, 497, 5, 226, 462, 495, 207, 497, 468, 493, 461, 495, 464, 497, 226, 462, 495, 207, 497, 468, 493, 461, 495, 464, 461, 495, 332, 497, 497, 42, 462, 495, 20, 497, 4, 493, 9, 493, 461, 495, 332, 497, 5, 497, 501])\n"
     ]
    }
   ],
   "source": [
    "print missing\n",
    "print \"Max length of sequence: \",max([len(x[1]) for x in set_list])\n",
    "## test\n",
    "print \"Testing!\"\n",
    "print buckets[random.choice(buckets.keys())][0]\n",
    "np.save(set+'_buckets',buckets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "properties = {}\n",
    "properties['vocab_size'] = len(vocab)\n",
    "properties['vocab'] = vocab\n",
    "properties['char_to_idx'] = char_to_idx\n",
    "properties['idx_to_char'] = idx_to_char\n",
    "import numpy as np\n",
    "np.save('properties',properties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I ( p ^ { 2 } , m _ { 2 } ^ { 2 } ) = N _ { d } ( - \\frac { 1 } { \\epsilon } + O ( \\epsilon ) ) ( 1 - \\frac { d } { 2 } ) \\int _ { 0 } ^ { 1 } d t ( \\frac { p ^ { 2 } } { m _ { 2 } ^ { 2 } } t ( 1 - t ) + 1 ) ^ { \\frac { d } { 2 } - 2 } ,\n",
      "\n",
      "\n",
      "\\alpha ( z ) = \\alpha _ { 0 } + \\tau \\alpha _ { 1 } + \\tau ^ { 2 } \\alpha _ { 2 } + \\ldots \\quad .\n",
      "\n",
      "\n",
      "{ \\cal L } = R - \\Lambda e ^ { a \\phi } - \\frac { 4 } { 3 } ( \\nabla \\phi ) ^ { 2 } - V e ^ { b \\phi } \\delta ( y )\n",
      "\n",
      "\n",
      "q _ { 6 } = \\frac 1 6 ( \\mathrm { t r } \\, M ) ^ { 3 } \\pm \\frac 1 2 \\mathrm { t r } \\, M \\, \\mathrm { t r } \\, M ^ { 2 } + \\frac 1 3 \\mathrm { t r } \\, M ^ { 3 } .\n",
      "\n",
      "\n",
      "\\Gamma _ { 0 } ^ { + } = - \\Gamma _ { 0 } , \\qquad \\Gamma _ { M } ^ { + } = \\Gamma _ { M } , \\qquad \\Gamma _ { M } ^ { T } = ( - 1 ) ^ { M + 1 } \\Gamma _ { M } .\n",
      "\n",
      "\n",
      "\\left( \\sum \\alpha _ { i } f _ { i } \\right) \\Rightarrow \\frac { 1 } { \\prod m _ { i } } \\sqrt { \\frac { \\Lambda ^ { ( N ) } } { D ^ { ( N ) } } } \\; \\gamma _ { N } = \\frac { 1 } { m _ { 0 } } \\; \\gamma _ { N } ,\n",
      "\n",
      "\n",
      "d i m ~ E _ { ( s ) } = 1 \\cdot \\phi \\cdot \\phi ^ { 2 } \\dot { . } . . \\phi ^ { s - 1 } = \\phi ^ { s ( s - 1 ) / 2 } ,\n",
      "\n",
      "\n",
      "\\begin{array} { c } { H _ { i j } = K _ { i j } = F _ { i j } ( i I ) , } \\\\ { D _ { i } ^ { A } \\Phi _ { A } = D _ { i } ^ { B } \\Phi _ { B } = E _ { i } ( i I ) . } \\\\ \\end{array}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f = open(\"im2markup/formulas.norm.lst\").readlines()\n",
    "np.random.seed(1234)\n",
    "import numpy as np\n",
    "for i in xrange(8):\n",
    "    print np.random.choice(f)\n",
    "    print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}