deep_learning/time_series_sequences/Bidirectional_Dynamic_Seq2Seq.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bidirectional Dynamic Seq2Seq\n",
    "\n",
    "Encoder is bidirectional now. It feeds previously generated tokens as inputs during training, instead of target sequence.\n",
    "\n",
    "Based on the repo https://github.com/ematvey/tensorflow-seq2seq-tutorials, some experience are noted:\n",
    "* Replacing projection (one-hot encoding followed by linear layer) with embedding (indexing weights of linear layer directly) is more efficient.\n",
    "* When decoding, feeding previously generated tokens as inputs adds robustness to model's errors. Nevertheless, feeding ground truth speeds up training. That is to say best practice is to mix both randomly while training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version: 1.11.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"tensorflow-seq2seq-tutorials\")\n",
    "import helpers\n",
    "\n",
    "import tensorflow as tf\n",
    "print(\"Tensorflow version: {}\".format(tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PAD = 0\n",
    "EOS = 1\n",
    "\n",
    "vocab_size = 10\n",
    "input_embedding_size = 20\n",
    "\n",
    "encoder_hidden_units = 20\n",
    "decoder_hidden_units = encoder_hidden_units * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "encoder_inputs = tf.placeholder(shape=(None, None), dtype=tf.int32, name=\"encoder_inputs\")\n",
    "encoder_inputs_length = tf.placeholder(shape=(None,), dtype=tf.int32, name=\"encoder_inputs_length\")\n",
    "\n",
    "decoder_targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name=\"decoder_targets\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We feed data via `decoder_inputs` to understand how it works in previous. Here we implement decoder with `tf.nn.raw_rnn` and construct `decoder_inputs` step by step."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embeddings = tf.get_variable(\"embeddings\", dtype=tf.float32, \\\n",
    "                             shape=[vocab_size, input_embedding_size], \\\n",
    "                             initializer=tf.random_uniform_initializer(-1.0, 1.0))\n",
    "\n",
    "encoder_inputs_embeded = tf.nn.embedding_lookup(embeddings, encoder_inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we replace un-directional `tf.nn.dynamic_rnn` with directional `tf.nn.bidirectional_dynamic_rnn` as the encoder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.rnn import LSTMCell, LSTMStateTuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "encoder_cell = LSTMCell(encoder_hidden_units)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "((encoder_fw_outputs, encoder_bw_outputs), (encoder_fw_final_state, encoder_bw_final_state)) = \\\n",
    "    tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_cell, \\\n",
    "                                    cell_bw=encoder_cell, \\\n",
    "                                    inputs=encoder_inputs_embeded, \\\n",
    "                                    sequence_length=encoder_inputs_length, \\\n",
    "                                    dtype=tf.float32, time_major=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'bidirectional_rnn/fw/fw/TensorArrayStack/TensorArrayGatherV3:0' shape=(?, ?, 20) dtype=float32>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_fw_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ReverseSequence:0' shape=(?, ?, 20) dtype=float32>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_bw_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTMStateTuple(c=<tf.Tensor 'bidirectional_rnn/fw/fw/while/Exit_3:0' shape=(?, 20) dtype=float32>, h=<tf.Tensor 'bidirectional_rnn/fw/fw/while/Exit_4:0' shape=(?, 20) dtype=float32>)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_fw_final_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTMStateTuple(c=<tf.Tensor 'bidirectional_rnn/bw/bw/while/Exit_3:0' shape=(?, 20) dtype=float32>, h=<tf.Tensor 'bidirectional_rnn/bw/bw/while/Exit_4:0' shape=(?, 20) dtype=float32>)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_bw_final_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have to concatenate forward and backward outputs and states. In the case we won't ignore outputs, they are useful for `attention` mechanism."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs), 2)\n",
    "\n",
    "encoder_final_state_c = tf.concat(\n",
    "    (encoder_fw_final_state.c, encoder_bw_final_state.c), 1)\n",
    "\n",
    "encoder_final_state_h = tf.concat(\n",
    "    (encoder_fw_final_state.h, encoder_bw_final_state.h), 1)\n",
    "\n",
    "encoder_final_state = LSTMStateTuple(\n",
    "    c=encoder_final_state_c,\n",
    "    h=encoder_final_state_h\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoder_cell = LSTMCell(decoder_hidden_units)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "encoder_max_time, batch_size = tf.unstack(tf.shape(encoder_inputs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Time and batch dimenions are dynamic. Next we decided how far to run the decoder, the stopping criteria are:\n",
    "* Stop after specified number of unrolling steps.\n",
    "* Stop after model produced token.\n",
    "\n",
    "The choice will likely be time-dependent. In legacy `translate` tutorial, the decoder unrolls for `len(encoder_input) + 10` to allow for possibly longer translated sequence. Here we demo a toy copy task so that we unroll decoder for `len(encoder_input) + 2` to allow some room to make minstakes over `2` additional steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 3 = 2 (additional steps) + 1 (leading <EOS> token)\n",
    "decoder_lengths = encoder_inputs_length + 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output projection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```text\n",
    "output(t) -> output projection(t) -> prediction(t) (argmax) -> input embedding(t+1) -> input(t+1)\n",
    "```\n",
    "\n",
    "We used `tf.contrib.layers.linear` layer to initialize weights and biases and apply operations for us. Here we need to specify parameters `W` and `b` of the output layer in global scope, and apply them at every step of the decoder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W = tf.Variable(tf.random_uniform([decoder_hidden_units, vocab_size], -1, 1), dtype=tf.float32)\n",
    "b = tf.Variable(tf.zeros([vocab_size]), dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder via `tf.nn.raw_rnn`\n",
    "\n",
    "`tf.nn.dynamic_rnn` allows for easy RNN construction, but it is limited.\n",
    "\n",
    "For example, a way to increase robustness of the model is to feed as decoder input tokens generated previously, instead of shifted true sequence.\n",
    "\n",
    "![](./tensorflow-seq2seq-tutorials/pictures/2-seq2seq-feed-previous.png)\n",
    "\n",
    "refer to http://www.wildml.com/2016/04/deep-learning-for-chatbots-part-1-introduction/."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assert EOS == 1 and PAD == 0\n",
    "\n",
    "eos_time_slice = tf.ones([batch_size], dtype=tf.int32, name='EOS')\n",
    "pad_time_slice = tf.zeros([batch_size], dtype=tf.int32, name='PAD')\n",
    "\n",
    "eos_step_embedded = tf.nn.embedding_lookup(embeddings, eos_time_slice)\n",
    "pad_step_embedded = tf.nn.embedding_lookup(embeddings, pad_time_slice)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The standard `tf.nn.dynamic_rnn` requires all inputs `(t, ..., t+n)` be passed as a single tensor. The `dynamic` means the fact that `n` can change from batch to batch.\n",
    "\n",
    "Now, what if we want to implement more complex mechanic like we want decoder to receive previously generated token as the input at every timestamp (instead of lagged target sequence)? Or when we want to implement a soft attention mechanism where at every timestamp we add additional fixed-length representation, derived from query produced by previous step's hidden state? We can use `tf.nn.raw_rnn` to solve these.\n",
    "\n",
    "Main part of specifying `tf.nn.raw_rnn` is `loop transition function`, defining input of step `t` given outputs and state of step `t-1`.\n",
    "\n",
    "`loop transition function` is called `before` RNNCell to prepare its inputs and states, and maps as below:\n",
    "```text\n",
    "(time, previous_cell_output, previous_cell_state, previous_loop_state) \n",
    "-> (elements_finished, input, cell_state, output, loop_state)\n",
    "```\n",
    "\n",
    "Everything is a Tensor except for initial call at time=0 where everything is `None` (except `time`).\n",
    "\n",
    "Loop transition function was called at two conditions:\n",
    "1. Initial call at time=0 to provide cell_state and input to RNN.\n",
    "2. Transition call for all following timestamps where you define transition between two adjacent steps."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loop initial state is function of only `encoder_final_state` and `embeddings`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loop_fn_initial():\n",
    "    initial_elements_finished = (0 >= decoder_lengths)  # all False at the initial step\n",
    "    initial_input = eos_step_embedded\n",
    "    initial_cell_state = encoder_final_state\n",
    "    initial_cell_output = None\n",
    "    initial_loop_state = None  # we don't need to pass any additional information\n",
    "    return (initial_elements_finished,\n",
    "            initial_input,\n",
    "            initial_cell_state,\n",
    "            initial_cell_output,\n",
    "            initial_loop_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define transition function that previously generated token is passed as next input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loop_fn_transition(time, previous_output, previous_state, previous_loop_state):\n",
    "\n",
    "    def get_next_input():\n",
    "        output_logits = tf.add(tf.matmul(previous_output, W), b)\n",
    "        prediction = tf.argmax(output_logits, axis=1)\n",
    "        next_input = tf.nn.embedding_lookup(embeddings, prediction)\n",
    "        return next_input\n",
    "    \n",
    "    elements_finished = (time >= decoder_lengths) # this operation produces boolean tensor of [batch_size]\n",
    "                                                  # defining if corresponding sequence has ended\n",
    "\n",
    "    finished = tf.reduce_all(elements_finished) # -> boolean scalar\n",
    "    input = tf.cond(finished, lambda: pad_step_embedded, get_next_input)\n",
    "    state = previous_state\n",
    "    output = previous_output\n",
    "    loop_state = None\n",
    "\n",
    "    return (elements_finished, \n",
    "            input,\n",
    "            state,\n",
    "            output,\n",
    "            loop_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine initializer and transition functions to create raw_rnn.\n",
    "\n",
    "All operations above are defined with TF's control flow and reduction ops, here we check whether state is `None` to determine if it is an initializer call or transition call."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def loop_fn(time, previous_output, previous_state, previous_loop_state):\n",
    "    if previous_state is None:    # time == 0\n",
    "        assert previous_output is None and previous_state is None\n",
    "        return loop_fn_initial()\n",
    "    else:\n",
    "        return loop_fn_transition(time, previous_output, previous_state, previous_loop_state)\n",
    "\n",
    "decoder_outputs_ta, decoder_final_state, _ = tf.nn.raw_rnn(decoder_cell, loop_fn)\n",
    "decoder_outputs = decoder_outputs_ta.stack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'TensorArrayStack/TensorArrayGatherV3:0' shape=(?, ?, 40) dtype=float32>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder_outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to do the output projection, we have to temporarily flatten decoder_outputs from `[max_steps, batch_size, hidden_dim]` to `[max_step*batch_size, hidden_dim]` (`tf.matmul`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoder_max_steps, decoder_batch_size, decoder_dim = tf.unstack(tf.shape(decoder_outputs))\n",
    "decoder_outputs_flat = tf.reshape(decoder_outputs, (-1, decoder_dim))\n",
    "decoder_logits_flat = tf.add(tf.matmul(decoder_outputs_flat, W), b)\n",
    "decoder_logits = tf.reshape(decoder_logits_flat, (decoder_max_steps, decoder_batch_size, vocab_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoder_prediction = tf.argmax(decoder_logits, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RNN projection layer map onto `[max_time, batch_size, vocab_size]`, and `vocab_size` of the shape is static, while `max_time` and `batch_size` are dynamic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-25-65256de115bf>:1: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stepwise_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(\\\n",
    "        labels=tf.one_hot(decoder_targets, depth=vocab_size, dtype=tf.float32),\\\n",
    "        logits=decoder_logits)\n",
    "\n",
    "loss = tf.reduce_mean(stepwise_cross_entropy)\n",
    "train_op = tf.train.AdamOptimizer().minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training on the toy task\n",
    "\n",
    "Consider the copy task - given the `random` sequence of integers from a vocabulary, learn to memorize and reproduce input sequence. Random sequence is generated in random, unlike natural language, it don't contain any structure."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data in batch: \n",
      "[8, 4, 2, 6]\n",
      "[2, 4, 5, 3, 2]\n",
      "[5, 8, 5, 8, 5]\n",
      "[2, 2, 3, 8]\n",
      "[9, 9, 3, 7]\n",
      "[6, 6, 3, 4, 4, 6]\n",
      "[2, 3, 6, 7, 7, 6]\n",
      "[8, 6, 4, 9, 7, 5, 5]\n",
      "[4, 4, 6, 5]\n",
      "[4, 2, 7, 5, 4, 3, 8, 8]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 100\n",
    "\n",
    "batches = helpers.random_sequences(length_from=3, length_to=8, vocab_lower=2, vocab_upper=10, batch_size=batch_size)\n",
    "\n",
    "print('data in batch: ')\n",
    "for seq in next(batches)[:10]:\n",
    "    print(seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def next_feed():\n",
    "    batch = next(batches)\n",
    "    encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch)\n",
    "    decoder_targets_, _ = helpers.batch(\n",
    "        [(sequence) + [EOS] + [PAD] * 2 for sequence in batch]\n",
    "    )\n",
    "    return {\n",
    "        encoder_inputs: encoder_inputs_,\n",
    "        encoder_inputs_length: encoder_input_lengths_,\n",
    "        decoder_targets: decoder_targets_,\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_history = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch 0\n",
      "  minibatch loss: 2.296715497970581\n",
      "  sample 1:\n",
      "    input     > [8 4 4 6 6 0 0 0]\n",
      "    predicted > [4 2 0 0 0 1 0 1 0 0 0]\n",
      "  sample 2:\n",
      "    input     > [9 4 9 3 3 7 0 0]\n",
      "    predicted > [4 4 6 4 6 2 6 2 9 0 0]\n",
      "  sample 3:\n",
      "    input     > [3 6 5 4 4 0 0 0]\n",
      "    predicted > [7 1 7 1 7 1 7 8 0 0 0]\n",
      "\n",
      "batch 1000\n",
      "  minibatch loss: 0.5229266881942749\n",
      "  sample 1:\n",
      "    input     > [8 3 2 0 0 0 0 0]\n",
      "    predicted > [8 3 2 1 0 0 0 0 0 0 0]\n",
      "  sample 2:\n",
      "    input     > [4 8 8 5 0 0 0 0]\n",
      "    predicted > [4 8 8 5 1 0 0 0 0 0 0]\n",
      "  sample 3:\n",
      "    input     > [8 2 5 4 0 0 0 0]\n",
      "    predicted > [8 2 5 4 1 0 0 0 0 0 0]\n",
      "\n",
      "batch 2000\n",
      "  minibatch loss: 0.23747214674949646\n",
      "  sample 1:\n",
      "    input     > [5 4 2 9 0 0 0 0]\n",
      "    predicted > [5 4 2 9 1 0 0 0 0 0 0]\n",
      "  sample 2:\n",
      "    input     > [6 8 3 4 8 6 0 0]\n",
      "    predicted > [6 8 3 4 8 6 1 0 0 0 0]\n",
      "  sample 3:\n",
      "    input     > [9 2 4 4 0 0 0 0]\n",
      "    predicted > [9 2 4 4 1 0 0 0 0 0 0]\n",
      "\n",
      "batch 3000\n",
      "  minibatch loss: 0.15095090866088867\n",
      "  sample 1:\n",
      "    input     > [5 4 4 4 2 5 3 9]\n",
      "    predicted > [5 4 4 4 2 5 3 9 1 0 0]\n",
      "  sample 2:\n",
      "    input     > [3 5 8 2 4 5 8 0]\n",
      "    predicted > [3 5 8 2 4 5 8 1 0 0 0]\n",
      "  sample 3:\n",
      "    input     > [4 4 2 8 3 0 0 0]\n",
      "    predicted > [4 4 2 8 3 1 0 0 0 0 0]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "max_batches = 3001\n",
    "batches_in_epoch = 1000\n",
    "\n",
    "try:\n",
    "    for batch in range(max_batches):\n",
    "        fd = next_feed()\n",
    "        _, l = sess.run([train_op, loss], fd)\n",
    "        loss_history.append(l)\n",
    "\n",
    "        if batch == 0 or batch % batches_in_epoch == 0:\n",
    "            print('batch {}'.format(batch))\n",
    "            print('  minibatch loss: {}'.format(sess.run(loss, fd)))\n",
    "            predict_ = sess.run(decoder_prediction, fd)\n",
    "            for i, (inp, pred) in enumerate(zip(fd[encoder_inputs].T, predict_.T)):\n",
    "                print('  sample {}:'.format(i + 1))\n",
    "                print('    input     > {}'.format(inp))\n",
    "                print('    predicted > {}'.format(pred))\n",
    "                if i >= 2:\n",
    "                    break\n",
    "            print()\n",
    "\n",
    "except KeyboardInterrupt:\n",
    "    print('training interrupted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.150731\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl4FeXd//H3NztZgEDCIlvYFxcEEXdRARG0olZb7KZ97EPdra39PaC1damVVq2t1Wqt0qpPXWqLj1RQFEVEBTEiIMgWVsMatiRA9ty/P84hBrId4CRzzpzP67pyMWfmzpnvzQkfJvfM3GPOOURExF/ivC5ARETCT+EuIuJDCncRER9SuIuI+JDCXUTEhxTuIiI+pHAXEfEhhbuIiA8p3EVEfCjBqx1nZWW5nJwcr3YvIhKVPvvss53Oueym2nkW7jk5OeTm5nq1exGRqGRmG0Npp2EZEREfUriLiPiQwl1ExIcU7iIiPqRwFxHxIYW7iIgPKdxFRHwo6sJ91bZiHp61it37y70uRUQkYkVduK/fuY/H5+SxrbDU61JERCJW1IV765REAApLKjyuREQkckVduLdKigegtKLK40pERCJX1IV7ckIg3MsqFe4iIg2JvnBPDJRcVlntcSUiIpEr+sI9QeEuItKUKAz3g8MyCncRkYZEX7gfHJbRCVURkQZFXbi3SownznQppIhIY6Iu3BPj48hKT6aguMzrUkREIlbUhTtAenIC+8oqvS5DRCRiRWe4pyjcRUQaE53hnpzAvlKFu4hIQ6I33HXkLiLSoOgMdw3LiIg0KirDPUNH7iIijYrKcE9LTmDvgQpWbC3yuhQRkYgUleGenpIAwNg/zvO4EhGRyBSV4b6jSDcwiYg0JirDvbJak4aJiDQmKsM9I/ioPRERqV+UhnuC1yWIiES0qAz3bw3r5nUJIiIRLSrDPSs92esSREQiWlSGO8Clg48DoLhU87qLiBwuasP9YKg/PGuVx5WIiESeqA33G8/vA0DvDukeVyIiEnmaDHcz62Zmc8zsSzNbbma31dPGzOwxM8szs6VmNrR5yv1ar6w0AJxr7j2JiESfUI7cK4GfOecGAacDN5nZoMPajAX6Br8mAk+Gtcp6HJyC4FfTlzf3rkREok6T4e6c2+qcWxRcLgZWAF0OazYeeN4FLADamlnnsFdbS3JCfM3y8i2FzbkrEZGoc0Rj7maWAwwBPjlsUxfgq1qv86n7H0CzKa3QdAQiIrWFHO5mlg78G/iJc+6o5to1s4lmlmtmuQUFBUfzFg3QwLuISG0hhbuZJRII9n8456bV02QzUPu20a7BdYdwzj3tnBvmnBuWnZ19NPXW6/n5G8P2XiIifhDK1TIGPAuscM79voFm04EfBK+aOR0odM5tDWOdjXp98RZKK6paanciIhEvlBm4zgK+D3xhZouD6+4EugM4554CZgLjgDzgAPDD8JfauOLSSlIS45tuKCISA5oMd+fch4A10cYBN4WrqFAN6d6WzzftBWBrYQnZGZpzRkQEovgOVYC//3A4owZ2AOClhZs8rkZEJHJEdbi3aZXIr75xPAAvLfyqidYiIrEjqsMd0FCMiEg9oj7cdRJVRKSuqA93gPEnH0eP9qlelyEiEjF8Ee6tUxIpKtFDO0REDvJHuLdKoKi0Eqf5f0VEAJ+Ee1Z6MlXVjq2FpV6XIiISEXwR7p1apwDw038ubqKliEhs8EW498wOPJVpwbrdHlciIhIZfBHuAzq19roEEZGI4otwry1/zwGvSxAR8Zzvwv2qp+Z7XYKIiOd8E+73jw/MMbOvtNLjSkREvOebcJ8wvDsAQ3tkelyJiIj3fBPuifGBrsxdHc5ns4qIRCffhHttVdW6U1VEYpsvw31rYYnXJYiIeMpX4T712mGAhmZERHwV7id0aQNAoWaIFJEY56twz05PJjUpnp3F5V6XIiLiKV+Fu5mRmZrEsi2FXpciIuKpBK8LCLfNe0vYvLcE5xxm5nU5IiKe8NWRe21FulNVRGKY78L9vuA0BHv2a9xdRGKX78K9W7vAg7J3KdxFJIb5LtzbpyUBsFvhLiIxzHfh3q4m3Ms8rkRExDu+C/es9GQAPSxbRGKa78I9JTEegD/MXuNxJSIi3vFduIuIiE/D/Zy+WQBUVlV7XImIiDd8Ge6tWyUCsHJbsceViIh4w5fhPmPpVgCmL9nicSUiIt7wZbj/6eohADz9wTqPKxER8UaT4W5mU81sh5kta2D7eWZWaGaLg1+/DH+ZR6ZnVprXJYiIeCqUWSH/DjwOPN9Im3nOuUvCUlEYVDs9Q1VEYluTR+7OuQ+A3S1QS9j06ZDudQkiIp4K15j7GWa2xMzeNLPjw/SeRy01KYGLju9Ex9bJXpciIuKJcDysYxHQwzm3z8zGAf8H9K2voZlNBCYCdO/ePQy7blhacgLbizS/jIjEpmM+cnfOFTnn9gWXZwKJZpbVQNunnXPDnHPDsrOzj3XXjfr3onwAlubvbdb9iIhEomMOdzPrZMHn2ZnZ8OB77jrW9w2X91cVeF2CiEiLC+VSyJeA+UB/M8s3s+vM7Hozuz7Y5EpgmZktAR4DJjjn/eUq028+C4An31/rcSUiIi2vyTF359zVTWx/nMClkhHlpK5tASipqKKiqprEeF/eryUiUq+YSDyNu4tIrImJcE9OiPe6BBGRFuXrcL9+RG8A3ghOJCYiEit8He4Hz+s+NVcnVUUktvg63A8euYuIxBpfh3tmWlLNsp7KJCKxxNfhXtsJ98yisKTC6zJERFqE78N96rXDACitqObvH23wthgRkRbi+3A/rWf7mmUduYtIrPB9uKclf30T7tSP1ntYiYhIy/F9uANkpiZ6XYKISIuKiXB/aeLpNcu795d7WImISMuIiXAf0Kl1zfLQ+9/xsBIRkZYRE+EOMPHcXjXL+8oqPaxERKT5xUy4D+2eWbP82cY9HlYiItL8YibcLzqhE9ed3ROAa6YuZI/G3kXEx2Im3AF6ZqXVLP/4hc88rEREpHnFVLhfPqRLzfLCDbs9rEREpHnFVLinJScwoFNGzetLH/+QCHjcq4hI2MVUuAN0apNSs7w0v5DSCs0WKSL+E3Ph/odvn3zI62c/XOdRJSIizSfmwr1tahIbplxc8/rht1fzh9mrPaxIRCT8Yi7cD5o0dkDN8h9mr+HjvJ0afxcR34jZcJ94Ti8yUr6eMfI7z3zCy59+5WFFIiLhE7PhHhdnvFxrQjGAydO+oKpaR+8iEv1iNtwBjj+uTZ11lz7+IdsKSz2oRkQkfGI63AH+fcMZh7xevqWIq/7ysUfViIiER8yH+yk92rH83jGHrPtqdwmvL97sUUUiIscu5sMdAneuHh7wt728mBlLt3pUkYjIsVG4B6UlJzC466Fj8De9uIh/fZbvUUUiIkdP4V7L6zefzYr7Ljpk3R2vLvGoGhGRo6dwP0yrpPg663ImzWDZ5kIPqhEROToK93rUnp7goEv+pBkkRSR6KNwb8Oldo+qs6zl5JgvW7fKgGhGRI6Nwb0B2RnKd8XeACU8v8KAaEZEj02S4m9lUM9thZssa2G5m9piZ5ZnZUjMbGv4yvVHf+DvA9iLdwSoikS2UI/e/A3UPYb82Fugb/JoIPHnsZUWOx78zpM66037zLm8s3UJpRRXFpRUeVCUi0riEpho45z4ws5xGmowHnneBs40LzKytmXV2zvniDqCzemfVu/7mFz+vWa7vBKyIiJfCMebeBag9V25+cJ0vZKYl8YuLBzLj1rM5v392vW10FY2IRJoWPaFqZhPNLNfMcgsKClpy18fkR+f04vjj2vC3Hw6v9yi95+SZlFVWeVCZiEj9whHum4FutV53Da6rwzn3tHNumHNuWHZ2/UfB0WD2T0fUWdf/F2+xYmuRB9WIiNQVjnCfDvwgeNXM6UChX8bbG9KnQzoL7xrJ7aP6HbJ+7B/nkTNpBjmTZuiKGhHxVCiXQr4EzAf6m1m+mV1nZteb2fXBJjOBdUAe8FfgxmarNoJ0yEjhtlF9G9x+2RMftWA1IiKHMq9OBg4bNszl5uZ6su9wmru6gB+/kEtpRXW921+9/gxOzWnXwlWJiF+Z2WfOuWFNtdMdqsdoRL9sVt4/tsHtVz01n0Wb9rRgRSIiCvew2TDlYn5x8UDS6rmr9Yo/f8w3n/yYaj18W0RaiMI9jH50Ti/+OKHuHa0An23cQ687Z7ZwRSISqxTuYTZqUEduOK93g9tfmL+Bh2atbLmCRCQmKdybwf9cNICFd42sd9vdry/niTlrW7giEYk1Cvdmkp2ezI9H9GLWT86lbWpine264UlEmpMuhWwBD81a2ejR+rJ7x5Ce3OQcbiIiuhQyktxyQV/uv+wEZt56Tr3bV2wt4sE3V5AzaQa795e3cHUi4kc6cvfAok17uOLPH9e7LTUpni/reQKUiAjoyD2iDe2e2eC2A+VVFB7QA0BE5Ngo3D3ygzN6NLht8H1vU1hSwdvLt7F+5/4WrEpE/EJn8Txy3/gTmLdmZ4PhPfjet2uWP797NJlpSS1Vmoj4gI7cPfTGLWeT+4tRvHbjmY22G3L/O8xfu6uFqhIRP9AJ1QhRVe3YXlTKmVPea7Tdt4d147dXntRCVYlIpNEJ1SgTH2cc17YVD1x+QqPtXsn9qtHtIiKgcI843z2tB2/eVv/18AflTJrB7a8s5kB5ZQtVJSLRRuEegQZ2bs1bP2k84F/7fDODfjmLUx+YzTPz1rVQZSISLRTuEWpAp9ZsmHIxj357MK9ef0aD7QqKy/j1jBV6ZquIHELhHuEuH9KVU3Pa8fndoxttd/sriyksqaCoVDdAiYiulok61dUupId+PPdfwzmnTxZxcdYCVYlIS9HVMj4VF2dsmHJxk+2umbqQXnfO5P1VOygpr2qBykQkkijco9ScO85j4rm9mmx37d8+5brnPm2BikQkkijco1TPrDTuHDeQN245u8m2H6/dxYNvrmiBqkQkUmjM3UdyJs0Iqd1rN57JkEZmphSRyBXqmLvC3WfeX7WDa//W9DDMxSd25vgurRk1sCM9s9IoragiI6Xu4wBFJLIo3GPY+p372XOgnGmL8vnfBZuabN8rO411BftZ/+A4zHR1jUgk09UyMaxnVhpDu2fy68tOZMmvLmyy/bqCwLTDU95aiXOOxV/tZeU2PcBbJJppPnefa50S+kf8l7nr+Mvcr6cyCOWSSxGJTAp3nzMzfjyiFyP6ZbPkq0LiDEYP6sgFj8xt8nvLKqtITohvgSpFJNwU7jFg8tiBAJzZOwsIzB0fiv6/eIt3fzaC3tnpzVabiDQPjbnHoPg4Y/ZPR7DwrpF0aduq0bYjH5nLT/+5mKLSCv7xyUa2Fpa0UJUicix0tYywaNMeXlu0mRcWbAyp/UNXnkSH1il0ap1C/04ZzVydiNQW6tUyGpYRhnbP5OSubUMO95//a2nN8oWDOvLk904BoLSiirRk/UiJRAIduUuNvQfKqap2HCivIjsjmQF3vxXS97VPS2LkwA78Mzeftb8ZR7xmohRpNrrOXY5Y29Qk2qcn061dKimJoV8ls2t/Of/MzQeg950zmbYon8feXdNcZYpICEI6cjezi4A/AvHAM865KYdtvxZ4CNgcXPW4c+6Zxt5TR+6Rr7CkgoLiMrLSk1i/cz8rthZz52tfhPz95/fP5nun92BEv2z2l1fRppWmNxA5VmEbczezeOAJYDSQD3xqZtOdc18e1vQV59zNR1WtRKQ2rRJrAnlI9ySGdM9ky94SnvlwHaUV1U1+/5xVBcxZVVDzur6bojbvLaF1SoLmtREJs1CGZYYDec65dc65cuBlYHzzliWR6o4x/Zl2w1kkxR/5iN4Ppi6sM63BWVPe47InPgpXeSISFMq/0C7AV7Ve5wfXHe6bZrbUzP5lZt3CUp1EpEHHtWb1A2PrrB/SvW2j3/fB6gIu+sM8DpRX8tTctew9UA7A2uDcNiISPuG6bu0/wEvOuTIz+zHwHHDB4Y3MbCIwEaB79+5h2rV4ZcOUi3HOUVnteHfFds7tlw3Ai59s4tczGn44yKBfzgJgypsra9Z9tnEP8XFGr+w0WiXGk3gUvxmIyNeaPKFqZmcA9zjnxgRfTwZwzj3YQPt4YLdzrk1j76sTqv5VUVVN37vePOrvH3tCJ4Z2z2T0oI7kZKWFsTKR6BfOSyE/BfqaWU8zSwImANMP21nnWi8vBfRMtxiWGB/Hiz867ai//81l23hg5grOe/h9cibNYMyjH3D4Qchj767hP0u2HGupIr7V5LCMc67SzG4GZhG4FHKqc265md0H5DrnpgO3mtmlQCWwG7i2GWuWKHBmnyzm/b/zmb5kCw/NWnVM77VqezE9J8/kpvN7c0avLPp1TOf376wG4JKTOusBIyL10B2q0iKcc0xbtJnB3dow6vcfhO193/3ZCL7IL+SyIV+f47/+hc+4YGAHvjVM5/XFf/SYPYlYFVXVNSdMq6sdizbt4cqn5h/z+069dhjn9+9Az8kzAfjJqL5cMaQr3dunHvN7i0QKhbtEnepqx9Pz1vHlliKyM5J59sP1YXnf20b2ZepH6/nwfy7QXbIS9RTu4gunPjCbguKysL3fOX2zePJ7p/DzV5dw76XH06F1StjeW6QlKNzFF0orqnj8vTwen5MXtvfs0yGdvB37GNi5NRNODYzLn5rTjmc/XM/dlwykbWoSO/eVEW9GZlpS2PYrEg4Kd/GV1xdv5raXFzf7fnpmpTHj1rNrbrRq6CHhc1buYHtRKROG62Y8aVl6WIf4yviTuzBqYEc27jqAwzGwU2veX72D7PQUTujSmmWbi/jG4x8e837W79xfE+wARaUVlFZU0SEjhcIDFaQkBU4E//DvnwJw8Umd2VFcpufMSsTRkbv4xh9nr+HR2avpkJFMWWU1hSUVYXvvRXePZuj979A6JYGi0sqa9Sd1bcPS/MIGj/BFwk3DMhKTtuwtoXOblJobm5xzNZdGNqeV9190RA84ETlaehKTxKTj2rY65I5VM+PfN5zJ1cO78eAVJ9asf+DyE7j7kkFh2+//LtjInJU7uPDRueRMmkFhSQXFpYHfHKqqHVXV3hxESezSkbvElHe+3E5maiLDctoBsGZ7MXk79nHDPxY1y/4uOakzyzYXsmHXATZMuZiqaseDM1dwYtc23DntC/aXV2lIR46ITqiK1GP0oI6HvO7bMYO+HTM4t182H6wuaOC7jt4bS7fWLP/p3TU8EpwTp7aXFm7i6uBVN4UlFbrRSsJCwzIiwDeHBuamuXp4N7Izkll+7xgWTB55yFDOsaov2AEmTws8l3bOqh0MvvdtFqzbBcDS/L3M/nJ7nfZllVVUVDX9mEOJbRqWEQkqq6wiOeHQk6LOOfaXV5GenEBhSQWPvrOazm1S6JqZyk0vhm8oZ9TAjsxe8XWQTzi1Gy9/GngAWptWibx245n0yk6nutrR686Z9GifyuSxA8lMTeS0Xu3rfc9pi/LpnZ3O4G4NPyHr4VmrGNA5g0tOOi5sfZHmpatlRJpRVbWj952Bq3CW/PJCxj02j817S5ptf0nxcZQ3cLS+9jfjiI87dNrjPfvLGXL/O0DDN2IB5Eya0WQbiSwacxdpRvFxxvt3nMfKbUW0SU3ko0kX1ARlc2go2AH2l1fSOuXrcfrD6/jZP5dw28i+XPHkx3Ruk8LrN53FGVPepXObVs1Wr3hPR+4iYVJZVU2cGXFxxurtxVz46AekJsXz09H9+NE5vSgoLuPUB2Y3aw2tEuMpqahqtM0T3xlaZ0gp74GxxJmxeW8J2RnJpCTGU1pRxfx1u+idlU739qmHTNUs3tGwjEgEWrmtiLSkBM753RyvS2nQuBM78efvnsJNLy5iRvBqn9tH9ePR2auZcevZLN9SxLbCUm4d2dfjSmOTwl0kgv3urZUUlVYwf+0uHrpqME+9v5bObVJ4bv5Gr0sDoFPrFLYVldZZn5YUz/7ywG8GL088nVNz2mHAgYrASeecSTO4fVQ/bhul4G8uCneRKHTN1IXMXV3AVad0ZdBxrfn+6T3YW1LBtEX5/GbmSgCevWYY1z0Xef92zurTno/yApdxfjzpAu6ZvpwfntWT3tlpvLhwE2sL9nPfpcdrGuVjpHAXiULvfLmd/34+l4V3jqzzIJHyymoWrt/N2X2zatat2lbMmD98UDOB2Wk925GVnkzXdq34wRk5PP7eGl5a+FVLd6NBKYlxPH71UOLi4K1l20hPTuTuSwZyy0uf88bSrdxyQR8uOek4+nfKCOt+q6sdZvjiYeoKd5EYUVFVTUKcUe2oc0kkQHFpBSfe87YHlYVmQKcMVm4rPmTdvZcez8iBHcjbsY+umamkJsXz7ortJCXE8a1h3Zi7uoDBXduG9FvA/rJKjv/VLH4+pj83nd8HCFwqump7Mac3cI9AJFO4i8ghrv3bQt5fVcCAThn07pDO6m3F/Oobx/Pl1sKaIZ9o89qNZ5IYH0f79CTOePA9rh/Rm66ZrRjesx0795Xxnb9+Quc2KWwtLKVDRjIL7xpF3o59jHtsHuWV1Uw8txc/H9OfPfvLj/qRi5VV1VQ5V+cGuOaicBeRI1ZYUkF8nFFZVc3UD9fz2Hvhe7xhJHj/jvM47+H369327s9GsK2wlO8+8wlz7jiPnPapvJqbzzcGH0erpLrBXV4Z+I3pm099zOeb9rLy/ov4avcB+nRIZ3tRGR+v3cmcVQU8ctVgkhLCdwmpbmISkSNWe9KyquCB309H92PM8Z149sN13DGmP/vLqujeLrXmDt1o0lCwA4x8ZG7N8vm12q3cVszkcQOodo6yymp2Fpdx84uf8+XWIkb0y+bzTXsBGHD3WwCMP/k4Xl+8peb7N+zcz+s3nYUDcjfspri0kuG92h1y41lzULiLSL26t0sFoEf7VPp3yuB3Vw4ObAie68xMDYTTw1cNpqC4jPEnd2FJ/l4mPL2Abw3ryuhBnfjv53PJzkjmye8O5cqn5je4rx7tU9m460Cz9udoTf1oPVM/Wl/vtrn1zCRaO9gBvthcSK/D/iM8t182z//X8PAVWQ8Ny4hIvZxzfLx2F2f2bl/vVSYHZ6Y8/K7VkvIqkhLi6pzcfe3zfKqq4am5a7nlgj58tnEP156ZwwWPzOWNW87mkj99/Qzcvh3SWbNjX83rK4Z2YdqizeHsnqcOjv8fDQ3LiMgxMTPO6pPV4PaGpiKob3wa4PIhXQG48pTAn+NPDkyzfHDSsoMzYa55YCwJcUbPyTM5NSeTV68/k1Xbipm2aDMXDOjAHyeczPurCti0+wAPzVp11P3zUrfgb0XNSUfuIhIRDmZRQ9ei79lfXufSxzXbixn96AeHrNsw5WLKK6u56cVF3D6qH699ns9f5wWGVTJTE9lzIHwPTj9aU644kQnBB7QcKV0tIyIxYcXWIrpmtmLvgQpWby9m5MCOddocvLIFYF7eTk7s0ob/LNnCxl0HWLW9iDgz5q3ZCcDT3z+Fc/pms6Ww5JCTrAB//u5Qbn5xEZ1ap7ClsO70DKFa95txxNVzT0IoFO4iIsfo7v9bxgsLNvLvG86gXVoyPbPSah7gsr5gPyd2bcPWwhJyN+whOyOZL7cUsWxzIfPX7eL20f0oKqngiTl5h/y28M7t59K349HfgatwFxGJEKu2FTN7xXZO6NKGEf2yj+m9dEJVRCRC9O+UEfb5cpqimfdFRHxI4S4i4kMKdxERH1K4i4j4UEjhbmYXmdkqM8szs0n1bE82s1eC2z8xs5xwFyoiIqFrMtzNLB54AhgLDAKuNrNBhzW7DtjjnOsDPAr8NtyFiohI6EI5ch8O5Dnn1jnnyoGXgfGHtRkPPBdc/hcw0vzwPCsRkSgVSrh3AWo/hDE/uK7eNs65SqAQqPP8KjObaGa5ZpZbUFB3qkwREQmPFr2JyTn3NPA0gJkVmNnGo3yrLGBn2ArzlvoSmfzSF7/0A9SXg3qE0iiUcN8MdKv1umtwXX1t8s0sAWgD7GrsTZ1zR30PrpnlhnL7bTRQXyKTX/ril36A+nKkQhmW+RToa2Y9zSwJmABMP6zNdOCa4PKVwHvOq0lrRESk6SN351ylmd0MzALiganOueVmdh+Q65ybDjwLvGBmecBuAv8BiIiIR0Iac3fOzQRmHrbul7WWS4Grwltao55uwX01N/UlMvmlL37pB6gvR8SzKX9FRKT5aPoBEREfirpwb2oqhEhkZhvM7AszW2xmucF17czsHTNbE/wzM7jezOyxYP+WmtlQD+ueamY7zGxZrXVHXLeZXRNsv8bMrqlvXx715R4z2xz8XBab2bha2yYH+7LKzMbUWu/pz5+ZdTOzOWb2pZktN7Pbguuj7nNppC/R+LmkmNlCM1sS7Mu9wfU9LTAlS54FpmhJCq5vcMqWhvp4xJxzUfNF4ITuWqAXkAQsAQZ5XVcIdW8Asg5b9ztgUnB5EvDb4PI44E3AgNOBTzys+1xgKLDsaOsG2gHrgn9mBpczI6Qv9wB31NN2UPBnKxnoGfyZi4+Enz+gMzA0uJwBrA7WG3WfSyN9icbPxYD04HIi8Enw7/ufwITg+qeAG4LLNwJPBZcnAK801sejqSnajtxDmQohWtSesuE54LJa6593AQuAtmbW2YsCnXMfELj6qbYjrXsM8I5zbrdzbg/wDnBR81d/qAb60pDxwMvOuTLn3Hogj8DPnuc/f865rc65RcHlYmAFgTvEo+5zaaQvDYnkz8U55/YFXyYGvxxwAYEpWaDu51LflC0N9fGIRVu4hzIVQiRywNtm9pmZTQyu6+ic2xpc3gYcfGR7pPfxSOuO9P7cHByumHpwKIMo6UvwV/khBI4So/pzOawvEIWfi5nFm9liYAeB/yzXAntdYEqWw+tqaMqWsPUl2sI9Wp3tnBtKYGbNm8zs3NobXeD3sai7bCla667lSaA3cDKwFXjE23JCZ2bpwL+Bnzjnimpvi7bPpZ6+ROXn4pyrcs6dTOAu/uHAAC/ribZwD2UqhIjjnNsc/HMH8BqBD377weGW4J87gs0jvY9HWnfE9sc5tz34D7Ia+Ctf//ob0X0xs0QCYfgP59y04Oqo/Fzq60u0fi4HOef2AnMm0qpcAAABZklEQVSAMwgMgx28n6h2XTU126FTtoStL9EW7qFMhRBRzCzNzDIOLgMXAss4dMqGa4DXg8vTgR8Er3I4HSis9et2JDjSumcBF5pZZvDX6wuD6zx32LmMywl8LhDoy4TgFQ09gb7AQiLg5y84LvsssMI59/tam6Luc2moL1H6uWSbWdvgcitgNIFzCHMITMkCdT+X+qZsaaiPR64lzyiH44vA2f/VBMaz7vK6nhDq7UXg7PcSYPnBmgmMr70LrAFmA+3c12fdnwj27wtgmIe1v0Tg1+IKAmN/1x1N3cB/ETgxlAf8MIL68kKw1qXBf1Sda7W/K9iXVcDYSPn5A84mMOSyFFgc/BoXjZ9LI32Jxs/lJODzYM3LgF8G1/ciEM55wKtAcnB9SvB1XnB7r6b6eKRfukNVRMSHom1YRkREQqBwFxHxIYW7iIgPKdxFRHxI4S4i4kMKdxERH1K4i4j4kMJdRMSH/j9PPHjw+UM16gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(loss_history)\n",
    "print(\"loss:{:4f}\".format(loss_history[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}