Frozen_lake_with_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-05-30 15:33:35,261] Making new env: FrozenLake-v0\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('FrozenLake-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_state = tf.placeholder(tf.float32,[1,16])\n",
    "W,B = tf.Variable(tf.random_uniform([16,4],0,0.02)),tf.Variable(0.2)\n",
    "all_Q_values =tf.add(tf.matmul(input_state,W),B)\n",
    "q_arg_max = tf.argmax(all_Q_values,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def state_gen(idx):\n",
    "    state = np.zeros([1,16])\n",
    "    state[0,idx] = 1\n",
    "    return state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Q_next = tf.placeholder(tf.float32,[1,4])\n",
    "cost = tf.reduce_sum(tf.square(Q_next - all_Q_values))\n",
    "opt = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy = []\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    reward_list = []\n",
    "    gamma = 0.99\n",
    "    num_episode = 2000\n",
    "    for i in xrange(num_episode):\n",
    "        s = env.reset()\n",
    "        d = False\n",
    "        step_reward = 0\n",
    "        for _ in xrange(100):\n",
    "            Q,a = sess.run([all_Q_values,q_arg_max],feed_dict={input_state:state_gen(s)})\n",
    "            #adding randomness\n",
    "            if np.random.uniform(0,1) < 0.1 and i < 7000:\n",
    "                a[0] = env.action_space.sample()\n",
    "            #taking action\n",
    "            s1,r,d,_ = env.step(a[0])\n",
    "            if(s1 == 5 or s1 == 7 or s1 == 11 or s1 == 12):\n",
    "                r = -2\n",
    "            Q1 = sess.run([all_Q_values],feed_dict={input_state:state_gen(s)})\n",
    "            Qmax = np.amax(Q1)\n",
    "            Q[0,a[0]] = r + gamma * Qmax\n",
    "            _,w = sess.run([opt,W],feed_dict={input_state:state_gen(s),Q_next:Q})\n",
    "            s = s1\n",
    "            step_reward += r\n",
    "            if d == True:\n",
    "                break\n",
    "        reward_list.append(step_reward)\n",
    "    \n",
    "    # display policy    \n",
    "    for itr in xrange(env.observation_space.n):\n",
    "        policy.append(sess.run([all_Q_values],feed_dict={input_state:state_gen(itr)}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state: 0\t value:\t1\n",
      "state: 1\t value:\t3\n",
      "state: 2\t value:\t3\n",
      "state: 3\t value:\t3\n",
      "state: 4\t value:\t0\n",
      "state: 5\t value:\t0\n",
      "state: 6\t value:\t0\n",
      "state: 7\t value:\t0\n",
      "state: 8\t value:\t3\n",
      "state: 9\t value:\t1\n",
      "state: 10\t value:\t0\n",
      "state: 11\t value:\t3\n",
      "state: 12\t value:\t3\n",
      "state: 13\t value:\t2\n",
      "state: 14\t value:\t3\n",
      "state: 15\t value:\t0\n"
     ]
    }
   ],
   "source": [
    "i=0\n",
    "for ele in policy:\n",
    "    print 'state: ' + str(i) + '\\t value:\\t' + str(ele[0].argmax())\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2035\n"
     ]
    }
   ],
   "source": [
    "rewd = 0\n",
    "for ele in reward_list:\n",
    "    if ele > 0:\n",
    "        rewd += ele\n",
    "print rewd/2000.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 0.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 0.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 0.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 0.0,\n",
       " 0.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 0.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 0.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 0.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " -2.0,\n",
       " 1.0,\n",
       " ...]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}