train_chars.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "\n",
    "from tensorflow.contrib.learn.python.learn.datasets import base\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os,sys\n",
    "sys.path.insert(0, './scripts')\n",
    "import py_compile\n",
    "py_compile.compile('scripts/ivector_tools.py')\n",
    "py_compile.compile('scripts/siamese_model_chars.py')\n",
    "import ivector_dataset\n",
    "import siamese_model_chars as siamese_model\n",
    "import ivector_tools as it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_dataset_size(dict_file,feat_file):\n",
    "# Counting feature dimension and total number of utterances\n",
    "    f = open(dict_file)\n",
    "    dict_dim = 0\n",
    "    for line in f:\n",
    "        dict_dim+=1\n",
    "    f.close()\n",
    "    feat_len = 0\n",
    "    f = open(feat_file)\n",
    "    for line in f:\n",
    "        feat_len+=1\n",
    "    f.close()\n",
    "    return dict_dim, feat_len\n",
    "\n",
    "def get_feat_label(dict_file, feat_file):\n",
    "# Get feature vectors from files\n",
    "    dict_dim, feat_len = get_dataset_size(dict_file,feat_file)\n",
    "    features = np.zeros((feat_len,dict_dim),dtype='float32')\n",
    "    labels = np.zeros((feat_len),dtype='int8')\n",
    "    names = []\n",
    "    f = open(feat_file)\n",
    "    count = 0\n",
    "    for line in f:\n",
    "        names.append(line.split()[0])\n",
    "        labels[count] = line.split()[1]\n",
    "        line= line.split()[2:]\n",
    "        for iter in range(0,len(line)):\n",
    "            elements = line[iter].split(':')\n",
    "            features[count][ int( elements[0] ) -1 ] = elements[1]\n",
    "        count = count + 1 \n",
    "    f.close()\n",
    "    \n",
    "    return features, labels, names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14000, 13285) (1524, 13285) (1492, 13285)\n"
     ]
    }
   ],
   "source": [
    "context = 3\n",
    "dict_file = 'data/train.vardial2017/dict.chars.c'+str(context)\n",
    "feat_file = 'data/train.vardial2017/chars.c'+str(context)\n",
    "trn_features, trn_labels, trn_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/dev.vardial2017/chars.c'+str(context)\n",
    "dev_features, dev_labels, dev_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/test.MGB3/chars.c'+str(context)\n",
    "tst_features, tst_labels, tst_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "print trn_features.shape, dev_features.shape, tst_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "languages = ['EGY','GLF','LAV','MSA','NOR']\n",
    "\n",
    "\n",
    "\n",
    "# load tst.MGB3 labels\n",
    "filename = 'data/test.MGB3/reference'\n",
    "tst_ref_names = np.loadtxt(filename,usecols=[0],dtype='string')\n",
    "tst_ref_labels = np.loadtxt(filename,usecols=[1],dtype='int')\n",
    "\n",
    "tst_labels_index = []\n",
    "for i,name in enumerate(tst_names):\n",
    "    for j, name_ref in enumerate(tst_ref_names):\n",
    "        if name == name_ref:\n",
    "            tst_labels_index = np.append(tst_labels_index,j)\n",
    "\n",
    "tst_labels = np.empty((np.size(tst_labels_index)))\n",
    "for i,j in enumerate(tst_labels_index):\n",
    "    tst_labels[i]=tst_ref_labels[int(j)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# merge trn+dev\n",
    "trndev_features = np.append(trn_features, dev_features,axis=0)\n",
    "trndev_labels = np.append(trn_labels,dev_labels)\n",
    "trndev_names = np.append(trn_names,dev_names)\n",
    "\n",
    "# #center and length norm.\n",
    "# m=np.mean(trn_features,axis=0)\n",
    "# A = np.cov(trn_features.transpose())\n",
    "# [a,D,V] = np.linalg.svd(A)\n",
    "# V= V.transpose()\n",
    "# W= np.dot(V, np.diag(1./( np.sqrt(D) + 0.0000000001 )))\n",
    "\n",
    "\n",
    "# trn_features = np.dot( np.subtract( trn_features, m), W)\n",
    "# trndev_features = np.dot( np.subtract( trndev_features, m), W)\n",
    "# dev_features = np.dot( np.subtract( dev_features, m), W)\n",
    "# tst_features = np.dot( np.subtract( tst_features, m), W)\n",
    "\n",
    "# trn_features = it.length_norm(trn_features.transpose()).transpose()\n",
    "# trndev_features = it.length_norm(trndev_features.transpose()).transpose()\n",
    "# dev_features = it.length_norm(dev_features.transpose()).transpose()\n",
    "# tst_features = it.length_norm(tst_features.transpose()).transpose()\n",
    "\n",
    "# trn_features = it.length_norm(trn_features)\n",
    "# trndev_features = it.length_norm(trndev_features)\n",
    "# dev_features = it.length_norm(dev_features)\n",
    "# tst_features = it.length_norm(tst_features)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((14000, 13285), (1524, 13285), (5, 13285), (1492, 13285))\n"
     ]
    }
   ],
   "source": [
    "#language modeling\n",
    "lang_mean=[]\n",
    "for i, lang in enumerate(languages):\n",
    "#     lang_mean.append(np.mean(np.append(trn_features[np.nonzero(trn_labels == i+1)] ,dev_features[np.nonzero(dev_labels == i+1)],axis=0),axis=0))\n",
    "    lang_mean.append(np.mean( trn_features[np.nonzero(trn_labels == i+1)][:],axis=0 ) )\n",
    "\n",
    "lang_mean = np.array(lang_mean)\n",
    "lang_mean = it.length_norm(lang_mean)\n",
    "\n",
    "print( np.shape(trn_features), np.shape(dev_features), np.shape(lang_mean),np.shape(tst_features) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline accurary on test dataset : 0.337\n",
      "Baseline accurary on dev dataset : 0.320\n"
     ]
    }
   ],
   "source": [
    "# Baseline performance on TST using CDS\n",
    "tst_scores = lang_mean.dot(tst_features.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Baseline accurary on test dataset : %0.3f' %(acc)\n",
    "\n",
    "# Baseline performance on DEV using CDS\n",
    "dev_scores = lang_mean.dot(dev_features.transpose() )\n",
    "hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "temp = ((dev_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(dev_labels))\n",
    "print 'Baseline accurary on dev dataset : %0.3f' %(acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((70000,), (70000,), (70000,))\n",
      "((14000, 13285), (14000, 13285), (14000,), (56000, 13285), (56000, 13285), (56000,))\n"
     ]
    }
   ],
   "source": [
    "# making pair of train i-vector with mean of each language i-vector\n",
    "#  example : for total 3 ivectors\n",
    "#  ivector   ivector_p  label\n",
    "#     1         1         1\n",
    "#     1         2         0\n",
    "#     1         3         0\n",
    "#     2         1         0\n",
    "#     2         2         1\n",
    "#     ...      ...       ...\n",
    "#     3         3         1\n",
    "\n",
    "# preparing pair labels\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(trn_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( trn_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# TRN dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "trn_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "trn_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "trn_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "trn_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((7620,), (7620,), (7620,))\n",
      "((1524, 13285), (1524, 13285), (1524,), (6096, 13285), (6096, 13285), (6096,))\n"
     ]
    }
   ],
   "source": [
    "# preparing pair labels of DEV dataset\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(dev_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( dev_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# DEV dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "dev_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "dev_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "dev_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "dev_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 88600)\n",
      "(?, 1500)\n",
      "(?, 600)\n",
      "(?, 88600)\n",
      "(?, 1500)\n",
      "(?, 600)\n"
     ]
    }
   ],
   "source": [
    "# init variables\n",
    "sess = tf.InteractiveSession()\n",
    "siamese = siamese_model.siamese(np.shape(trn_features)[1]);\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learning_rate = tf.train.exponential_decay(0.01, global_step,\n",
    "                                           5000, 0.99, staircase=True)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(siamese.loss, global_step=global_step)\n",
    "saver = tf.train.Saver()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0: loss 1.002009, Acc.: (DEV)0.253 (TST)0.238, lr : 0.01000\n",
      "Step 100: loss 1.000225, Acc.: (DEV)0.253 (TST)0.249, lr : 0.01000\n",
      "Step 5100: loss 1.000026, Acc.: (DEV)0.257 (TST)0.250, lr : 0.00990\n",
      "Step 5600: loss 0.999658, Acc.: (DEV)0.262 (TST)0.255, lr : 0.00990\n",
      "Step 5800: loss 1.000857, Acc.: (DEV)0.274 (TST)0.260, lr : 0.00990\n",
      "Step 7100: loss 0.999062, Acc.: (DEV)0.283 (TST)0.275, lr : 0.00990\n",
      "Step 7700: loss 0.997398, Acc.: (DEV)0.290 (TST)0.288, lr : 0.00990\n",
      "Step 9000: loss 0.997168, Acc.: (DEV)0.295 (TST)0.289, lr : 0.00990\n",
      "Step 9600: loss 0.994928, Acc.: (DEV)0.295 (TST)0.293, lr : 0.00990\n",
      "Step 11000: loss 0.995609, Acc.: (DEV)0.301 (TST)0.294, lr : 0.00980\n",
      "Step 11100: loss 0.989397, Acc.: (DEV)0.293 (TST)0.296, lr : 0.00980\n",
      "Step 11500: loss 0.991541, Acc.: (DEV)0.298 (TST)0.296, lr : 0.00980\n",
      "Step 11600: loss 0.976284, Acc.: (DEV)0.296 (TST)0.297, lr : 0.00980\n",
      "Step 12200: loss 0.999093, Acc.: (DEV)0.303 (TST)0.300, lr : 0.00980\n",
      "Step 12700: loss 0.982434, Acc.: (DEV)0.305 (TST)0.302, lr : 0.00980\n",
      "Step 12800: loss 0.994154, Acc.: (DEV)0.308 (TST)0.302, lr : 0.00980\n",
      "Step 13000: loss 0.991245, Acc.: (DEV)0.311 (TST)0.304, lr : 0.00980\n",
      "Step 13100: loss 1.028808, Acc.: (DEV)0.313 (TST)0.306, lr : 0.00980\n",
      "Step 13300: loss 0.972647, Acc.: (DEV)0.313 (TST)0.308, lr : 0.00980\n",
      "Step 13500: loss 0.997523, Acc.: (DEV)0.312 (TST)0.311, lr : 0.00980\n",
      "Step 14400: loss 1.007398, Acc.: (DEV)0.320 (TST)0.314, lr : 0.00980\n",
      "Step 14500: loss 0.991952, Acc.: (DEV)0.319 (TST)0.316, lr : 0.00980\n",
      "Step 14900: loss 1.008784, Acc.: (DEV)0.324 (TST)0.322, lr : 0.00980\n",
      "Step 16400: loss 1.018120, Acc.: (DEV)0.341 (TST)0.323, lr : 0.00970\n",
      "Step 17100: loss 0.959200, Acc.: (DEV)0.348 (TST)0.324, lr : 0.00970\n",
      "Step 17700: loss 0.984836, Acc.: (DEV)0.349 (TST)0.325, lr : 0.00970\n",
      "Step 18000: loss 1.006642, Acc.: (DEV)0.354 (TST)0.327, lr : 0.00970\n",
      "Step 18400: loss 0.975814, Acc.: (DEV)0.363 (TST)0.333, lr : 0.00970\n",
      "Step 19300: loss 0.975327, Acc.: (DEV)0.367 (TST)0.335, lr : 0.00970\n",
      "Step 20200: loss 0.943386, Acc.: (DEV)0.384 (TST)0.337, lr : 0.00961\n",
      "Step 21600: loss 0.939106, Acc.: (DEV)0.392 (TST)0.340, lr : 0.00961\n",
      "Step 21800: loss 0.996622, Acc.: (DEV)0.412 (TST)0.343, lr : 0.00961\n",
      "Step 21900: loss 0.976396, Acc.: (DEV)0.406 (TST)0.345, lr : 0.00961\n",
      "Step 22600: loss 0.950466, Acc.: (DEV)0.422 (TST)0.352, lr : 0.00961\n",
      "Step 23000: loss 0.971509, Acc.: (DEV)0.433 (TST)0.357, lr : 0.00961\n",
      "Step 24100: loss 0.966037, Acc.: (DEV)0.432 (TST)0.359, lr : 0.00961\n",
      "Step 24400: loss 0.965899, Acc.: (DEV)0.444 (TST)0.359, lr : 0.00961\n",
      "Step 24500: loss 0.910001, Acc.: (DEV)0.432 (TST)0.366, lr : 0.00961\n",
      "Step 25100: loss 0.967876, Acc.: (DEV)0.448 (TST)0.367, lr : 0.00951\n",
      "Step 26500: loss 0.956474, Acc.: (DEV)0.497 (TST)0.370, lr : 0.00951\n",
      "Step 26700: loss 0.901570, Acc.: (DEV)0.482 (TST)0.372, lr : 0.00951\n",
      "Step 26800: loss 0.964571, Acc.: (DEV)0.520 (TST)0.381, lr : 0.00951\n",
      "Step 27500: loss 0.938040, Acc.: (DEV)0.530 (TST)0.397, lr : 0.00951\n",
      "Step 28100: loss 0.932819, Acc.: (DEV)0.604 (TST)0.398, lr : 0.00951\n",
      "Step 28200: loss 0.949939, Acc.: (DEV)0.629 (TST)0.403, lr : 0.00951\n",
      "Step 28500: loss 1.013062, Acc.: (DEV)0.591 (TST)0.404, lr : 0.00951\n",
      "Step 28600: loss 0.879185, Acc.: (DEV)0.635 (TST)0.412, lr : 0.00951\n",
      "Step 28700: loss 0.893499, Acc.: (DEV)0.627 (TST)0.420, lr : 0.00951\n",
      "Step 28800: loss 0.996071, Acc.: (DEV)0.669 (TST)0.432, lr : 0.00951\n",
      "Step 28900: loss 0.969667, Acc.: (DEV)0.656 (TST)0.436, lr : 0.00951\n",
      "Step 29200: loss 0.907319, Acc.: (DEV)0.662 (TST)0.439, lr : 0.00951\n",
      "Step 29500: loss 0.950047, Acc.: (DEV)0.680 (TST)0.446, lr : 0.00951\n",
      "Step 30300: loss 0.882984, Acc.: (DEV)0.703 (TST)0.450, lr : 0.00941\n",
      "Step 30500: loss 0.925474, Acc.: (DEV)0.719 (TST)0.478, lr : 0.00941\n",
      "Step 30800: loss 0.946772, Acc.: (DEV)0.705 (TST)0.480, lr : 0.00941\n",
      "Step 30900: loss 0.979380, Acc.: (DEV)0.744 (TST)0.485, lr : 0.00941\n",
      "Step 32100: loss 0.942349, Acc.: (DEV)0.774 (TST)0.494, lr : 0.00941\n",
      "Step 33100: loss 0.902451, Acc.: (DEV)0.825 (TST)0.509, lr : 0.00941\n",
      "Step 35600: loss 0.870834, Acc.: (DEV)0.848 (TST)0.511, lr : 0.00932\n",
      "Step 36100: loss 0.974957, Acc.: (DEV)0.909 (TST)0.521, lr : 0.00932\n",
      "Step 37300: loss 0.845368, Acc.: (DEV)0.923 (TST)0.523, lr : 0.00932\n",
      "Step 38100: loss 0.921455, Acc.: (DEV)0.912 (TST)0.529, lr : 0.00932\n",
      "Step 39300: loss 0.852358, Acc.: (DEV)0.950 (TST)0.530, lr : 0.00932\n",
      "Step 39700: loss 0.842584, Acc.: (DEV)0.945 (TST)0.532, lr : 0.00932\n",
      "Step 40000: loss 0.928585, Acc.: (DEV)0.958 (TST)0.534, lr : 0.00923\n",
      "Step 41000: loss 0.884517, Acc.: (DEV)0.964 (TST)0.542, lr : 0.00923\n",
      "Step 41200: loss 0.880605, Acc.: (DEV)0.974 (TST)0.551, lr : 0.00923\n",
      "Step 45200: loss 0.823282, Acc.: (DEV)0.984 (TST)0.554, lr : 0.00914\n",
      "Step 46200: loss 0.824312, Acc.: (DEV)0.993 (TST)0.556, lr : 0.00914\n",
      "Step 49700: loss 0.779730, Acc.: (DEV)0.999 (TST)0.558, lr : 0.00914\n",
      "Step 50000: loss 0.915867, Acc.: (DEV)0.999 (TST)0.560, lr : 0.00904\n",
      "Step 51200: loss 0.760859, Acc.: (DEV)1.000 (TST)0.562, lr : 0.00904\n",
      "Step 53200: loss 0.753681, Acc.: (DEV)1.000 (TST)0.563, lr : 0.00904\n",
      "Step 54100: loss 0.791382, Acc.: (DEV)1.000 (TST)0.564, lr : 0.00904\n",
      "Step 54600: loss 0.778552, Acc.: (DEV)1.000 (TST)0.569, lr : 0.00904\n",
      "Step 59300: loss 0.766740, Acc.: (DEV)1.000 (TST)0.570, lr : 0.00895\n",
      "Step 62100: loss 0.739804, Acc.: (DEV)1.000 (TST)0.572, lr : 0.00886\n",
      "Step 64500: loss 0.712104, Acc.: (DEV)1.000 (TST)0.575, lr : 0.00886\n",
      "Step 67900: loss 0.683895, Acc.: (DEV)1.000 (TST)0.576, lr : 0.00878\n",
      "Step 68800: loss 0.626072, Acc.: (DEV)1.000 (TST)0.576, lr : 0.00878\n",
      "Step 71600: loss 0.630243, Acc.: (DEV)1.000 (TST)0.582, lr : 0.00869\n"
     ]
    }
   ],
   "source": [
    "#start training\n",
    "batch_size = 40\n",
    "max_acc = 0.0\n",
    "max_step=0\n",
    "saver_folder='snnmodel_chars'\n",
    "if not os.path.exists(saver_folder):\n",
    "    os.mkdir(saver_folder)\n",
    "for step in range(200000):\n",
    "  \n",
    "    if step %5 ==0:\n",
    "        batch_x1_a, batch_y1_a = trn_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = trn_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = trn_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = trn_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "    else:\n",
    "        batch_x1_a, batch_y1_a = dev_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = dev_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = dev_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = dev_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "\n",
    "    batch_y = batch_y1*2-1\n",
    "    \n",
    "    _, loss_v = sess.run([train_step, siamese.loss], feed_dict={\n",
    "        siamese.x1: batch_x1,\n",
    "        siamese.x2: batch_x2,\n",
    "        siamese.y_: batch_y\n",
    "    })\n",
    "    \n",
    "    if np.isnan(loss_v):\n",
    "        print ('Model diverged with loss = NAN')\n",
    "        quit()\n",
    "        \n",
    "    if step % 100 ==0:\n",
    "        dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "        lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "        tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "        \n",
    "        dev_scores = lang_mean_siam.dot(dev_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "        temp = ((dev_labels-1) - hypo_lang)\n",
    "        acc =1- np.size(np.nonzero(temp)) / float( np.size(dev_labels) )\n",
    "        \n",
    "        tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "        temp = ((tst_labels-1) - hypo_lang)\n",
    "        acc_tst =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "\n",
    "        if max_acc < acc_tst:\n",
    "            max_acc = acc_tst\n",
    "            max_step=step\n",
    "            print ('Step %d: loss %f, Acc.: (DEV)%.3f (TST)%.3f, lr : %.5f' % (step,loss_v,acc,acc_tst,sess.run(learning_rate)))\n",
    "            saver.save(sess, saver_folder+'/model'+str(step)+'.ckpt')\n",
    "        if loss_v<0.3:\n",
    "            break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71600\n",
      "INFO:tensorflow:Restoring parameters from snnmodel_chars/model71600.ckpt\n",
      "The history saving thread hit an unexpected error (OperationalError('disk I/O error',)).History will not be written to the database.\n",
      "Final accurary on test dataset : 0.582\n"
     ]
    }
   ],
   "source": [
    "print max_step\n",
    "RESTORE_STEP=max_step\n",
    "saver.restore(sess, saver_folder+'/model'+str(RESTORE_STEP)+'.ckpt')\n",
    "\n",
    "\n",
    "# trn_features_siam = siamese.o1.eval({siamese.x1:trn_features})\n",
    "dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "\n",
    "tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Final accurary on test dataset : %0.3f' %(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix\n",
      "[[ 170.   32.   32.   11.   34.]\n",
      " [  30.  108.   30.   24.   35.]\n",
      " [  57.   44.  214.   24.   50.]\n",
      " [  17.   32.   16.  175.   24.]\n",
      " [  28.   34.   42.   28.  201.]]\n",
      "Precision\n",
      "[ 0.609319    0.47577093  0.55012853  0.66287879  0.6036036 ]\n",
      "Recall\n",
      "[ 0.56291391  0.432       0.64071856  0.66793893  0.58430233]\n",
      "\n",
      "\n",
      "<Performance evaluation on Test dataset>\n",
      "Accurary  : 0.582\n",
      "Precision : 0.580\n",
      "Recall    : 0.578\n"
     ]
    }
   ],
   "source": [
    "confusionmat = np.zeros((5,5))\n",
    "for i,lang in enumerate(languages):\n",
    "    hypo_bylang = hypo_lang[ tst_labels == i+1]\n",
    "    hist_bylang = np.histogram(hypo_bylang,5)\n",
    "    confusionmat[:,i] = hist_bylang[0]\n",
    "\n",
    "precision = np.diag(confusionmat) / np.sum(confusionmat,axis=1) #precision\n",
    "recall = np.diag(confusionmat) / np.sum(confusionmat,axis=0) # recall\n",
    "    \n",
    "print 'Confusion matrix'\n",
    "print confusionmat\n",
    "print 'Precision'\n",
    "print precision\n",
    "print 'Recall'\n",
    "print recall\n",
    "\n",
    "print '\\n\\n<Performance evaluation on Test dataset>'\n",
    "print 'Accurary  : %0.3f' %(acc)\n",
    "print 'Precision : %0.3f' %(np.mean(precision))\n",
    "print 'Recall    : %0.3f' %(np.mean(recall))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}