final_exp_norm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "generous-trailer",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import os\n",
    "import cv2\n",
    "from tqdm.notebook import trange\n",
    "\n",
    "import gym\n",
    "from gym import spaces\n",
    "\n",
    "from stable_baselines3 import DQN\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.dqn.policies import CnnPolicy\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import sklearn as sk\n",
    "from sklearn import datasets\n",
    "# from stable_baselines3.common.policies import register_policy\n",
    "# register_policy(\"myPolicy\", CnnPolicy)\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "noticed-modification",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('/vol1/idrl/idrl_env/mmd/idrl_mmd/')\n",
    "norm_models = sorted(glob('*norm*.zip'))\n",
    "models = sorted(glob('*.zip'))\n",
    "models = [model for model in models if model not in norm_models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "alleged-greeting",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_svmlight_file\n",
    "from dqn import *\n",
    "import numpy as np\n",
    "from stable_baselines3 import DQN\n",
    "\n",
    "\n",
    "train_norm = load_svmlight_file('data/mnist_data')\n",
    "x_norm, y_norm = train\n",
    "x_norm = x_norm.todense()\n",
    "\n",
    "sortind = np.argsort(y_norm)\n",
    "x_norm = x_norm[sortind, :]\n",
    "y_norm = y_norm[sortind]\n",
    "\n",
    "test_norm = load_svmlight_file('data/mnist_data.t')\n",
    "testx_norm, testy_norm = test_norm\n",
    "testx_norm = testx_norm.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "mathematical-arbitration",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dqn_cnn_mnist_norm_100_tarun.zip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d734276893194c549ed7489cd97b299a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 66.75%\n",
      "dqn_cnn_mnist_norm_10_tarun.zip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b1569d070f7400ba2f38aa3d68e3692",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 54.57%\n",
      "dqn_cnn_mnist_norm_1_tarun.zip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7efc928da9149a5a083bf38e9effae8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 11.47%\n",
      "dqn_cnn_mnist_norm_25_tarun.zip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86a40723846448f68ccf5e2068ebb13f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 67.20%\n",
      "dqn_cnn_mnist_norm_50_tarun.zip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd6cfa8962ab49c6b7113f8ffc1b9c49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 62.63%\n",
      "dqn_cnn_mnist_norm_5_tarun.zip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4ac2a76383a495abbf34a813b9c7a9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 47.32%\n",
      "dqn_cnn_mnist_norm_75_tarun.zip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bfe8a3d49e44f1bb22d0061bbe64e8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 67.92%\n"
     ]
    }
   ],
   "source": [
    "width, height = 64, 64\n",
    "for model_name in norm_models:\n",
    "    print(model_name)\n",
    "    model = DQN.load(model_name)\n",
    "    \n",
    "    total_rewards = 0\n",
    "    pred_y_test = []\n",
    "    count = 0\n",
    "\n",
    "    for idx in trange(len(testx_norm)):\n",
    "        # Generate an evaluation observation frame.\n",
    "        obs = cv2.resize(np.array(testx_norm[idx]).astype(np.float32), (width, height), interpolation = cv2.INTER_CUBIC)\n",
    "        obs = obs.reshape(width, height, 1)\n",
    "\n",
    "        # Predict an action based on the observation.\n",
    "        action, _states = model.predict(obs)\n",
    "        pred_y_test.append(action)\n",
    "\n",
    "\n",
    "        # Score the prediction.\n",
    "        if (action.size > 1):\n",
    "            # print(action)\n",
    "            count += 1\n",
    "            pass\n",
    "        else:\n",
    "            reward = 1 if action == testy_norm[idx] else 0\n",
    "            total_rewards += reward\n",
    "\n",
    "    print('Accuracy: {:.2f}%'.format(total_rewards / (len(testy_norm) - count) * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "editorial-sector",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.962    |\n",
      "| time/               |          |\n",
      "|    episodes         | 4        |\n",
      "|    fps              | 8651     |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total timesteps  | 6000     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0431   |\n",
      "|    n_updates        | 3651     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.924    |\n",
      "| time/               |          |\n",
      "|    episodes         | 8        |\n",
      "|    fps              | 8762     |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total timesteps  | 12000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.886    |\n",
      "| time/               |          |\n",
      "|    episodes         | 12       |\n",
      "|    fps              | 8697     |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total timesteps  | 18000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.848    |\n",
      "| time/               |          |\n",
      "|    episodes         | 16       |\n",
      "|    fps              | 8495     |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total timesteps  | 24000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.81     |\n",
      "| time/               |          |\n",
      "|    episodes         | 20       |\n",
      "|    fps              | 8603     |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total timesteps  | 30000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.772    |\n",
      "| time/               |          |\n",
      "|    episodes         | 24       |\n",
      "|    fps              | 8671     |\n",
      "|    time_elapsed     | 4        |\n",
      "|    total timesteps  | 36000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.734    |\n",
      "| time/               |          |\n",
      "|    episodes         | 28       |\n",
      "|    fps              | 8776     |\n",
      "|    time_elapsed     | 4        |\n",
      "|    total timesteps  | 42000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.696    |\n",
      "| time/               |          |\n",
      "|    episodes         | 32       |\n",
      "|    fps              | 8779     |\n",
      "|    time_elapsed     | 5        |\n",
      "|    total timesteps  | 48000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.658    |\n",
      "| time/               |          |\n",
      "|    episodes         | 36       |\n",
      "|    fps              | 1459     |\n",
      "|    time_elapsed     | 37       |\n",
      "|    total timesteps  | 54000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.038    |\n",
      "|    n_updates        | 999      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.62     |\n",
      "| time/               |          |\n",
      "|    episodes         | 40       |\n",
      "|    fps              | 731      |\n",
      "|    time_elapsed     | 81       |\n",
      "|    total timesteps  | 60000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0115   |\n",
      "|    n_updates        | 2499     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.582    |\n",
      "| time/               |          |\n",
      "|    episodes         | 44       |\n",
      "|    fps              | 519      |\n",
      "|    time_elapsed     | 126      |\n",
      "|    total timesteps  | 66000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0399   |\n",
      "|    n_updates        | 3999     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.544    |\n",
      "| time/               |          |\n",
      "|    episodes         | 48       |\n",
      "|    fps              | 428      |\n",
      "|    time_elapsed     | 168      |\n",
      "|    total timesteps  | 72000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0573   |\n",
      "|    n_updates        | 5499     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.506    |\n",
      "| time/               |          |\n",
      "|    episodes         | 52       |\n",
      "|    fps              | 364      |\n",
      "|    time_elapsed     | 213      |\n",
      "|    total timesteps  | 78000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0211   |\n",
      "|    n_updates        | 6999     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.468    |\n",
      "| time/               |          |\n",
      "|    episodes         | 56       |\n",
      "|    fps              | 326      |\n",
      "|    time_elapsed     | 257      |\n",
      "|    total timesteps  | 84000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0303   |\n",
      "|    n_updates        | 8499     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.43     |\n",
      "| time/               |          |\n",
      "|    episodes         | 60       |\n",
      "|    fps              | 296      |\n",
      "|    time_elapsed     | 303      |\n",
      "|    total timesteps  | 90000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0184   |\n",
      "|    n_updates        | 9999     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.392    |\n",
      "| time/               |          |\n",
      "|    episodes         | 64       |\n",
      "|    fps              | 275      |\n",
      "|    time_elapsed     | 349      |\n",
      "|    total timesteps  | 96000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0137   |\n",
      "|    n_updates        | 11499    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.354    |\n",
      "| time/               |          |\n",
      "|    episodes         | 68       |\n",
      "|    fps              | 257      |\n",
      "|    time_elapsed     | 395      |\n",
      "|    total timesteps  | 102000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0409   |\n",
      "|    n_updates        | 12999    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.316    |\n",
      "| time/               |          |\n",
      "|    episodes         | 72       |\n",
      "|    fps              | 243      |\n",
      "|    time_elapsed     | 443      |\n",
      "|    total timesteps  | 108000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0169   |\n",
      "|    n_updates        | 14499    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.278    |\n",
      "| time/               |          |\n",
      "|    episodes         | 76       |\n",
      "|    fps              | 233      |\n",
      "|    time_elapsed     | 487      |\n",
      "|    total timesteps  | 114000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0244   |\n",
      "|    n_updates        | 15999    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.24     |\n",
      "| time/               |          |\n",
      "|    episodes         | 80       |\n",
      "|    fps              | 223      |\n",
      "|    time_elapsed     | 536      |\n",
      "|    total timesteps  | 120000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0206   |\n",
      "|    n_updates        | 17499    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.202    |\n",
      "| time/               |          |\n",
      "|    episodes         | 84       |\n",
      "|    fps              | 214      |\n",
      "|    time_elapsed     | 586      |\n",
      "|    total timesteps  | 126000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0093   |\n",
      "|    n_updates        | 18999    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration rate | 0.164    |\n",
      "| time/               |          |\n",
      "|    episodes         | 88       |\n",
      "|    fps              | 207      |\n",
      "|    time_elapsed     | 636      |\n",
      "|    total timesteps  | 132000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0237   |\n",
      "|    n_updates        | 20499    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-d2442a3f50cb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mtest_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDQN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCnnPolicy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mtest_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_norm\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mtest_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'models/norm_{}_{}.zip'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtestm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/dqn.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    232\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0meval_log_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_log_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             \u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m         )\n\u001b[1;32m    236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    257\u001b[0m                 \u001b[0mlearning_starts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearning_starts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m                 \u001b[0mreplay_buffer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplay_buffer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m                 \u001b[0mlog_interval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlog_interval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    260\u001b[0m             )\n\u001b[1;32m    261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mcollect_rollouts\u001b[0;34m(self, env, callback, n_episodes, n_steps, action_noise, learning_starts, replay_buffer, log_interval)\u001b[0m\n\u001b[1;32m    404\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m                 \u001b[0;31m# Select action randomly or according to policy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 406\u001b[0;31m                 \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer_action\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sample_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearning_starts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction_noise\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    407\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m                 \u001b[0;31m# Rescale and perform action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36m_sample_action\u001b[0;34m(self, learning_starts, action_noise)\u001b[0m\n\u001b[1;32m    305\u001b[0m             \u001b[0;31m# we assume that the policy uses tanh to scale the action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m             \u001b[0;31m# We use non-deterministic action in the case of SAC, for TD3, it does not matter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m             \u001b[0munscaled_action\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_obs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m         \u001b[0;31m# Rescale the action from [low, high] to [-1, 1]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/dqn.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, mask, deterministic)\u001b[0m\n\u001b[1;32m    207\u001b[0m             \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m             \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, mask, deterministic)\u001b[0m\n\u001b[1;32m    285\u001b[0m         \u001b[0mobservation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m             \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m         \u001b[0;31m# Convert to numpy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m         \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/policies.py\u001b[0m in \u001b[0;36m_predict\u001b[0;34m(self, obs, deterministic)\u001b[0m\n\u001b[1;32m    167\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mq_net\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/policies.py\u001b[0m in \u001b[0;36m_predict\u001b[0;34m(self, observation, deterministic)\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobservation\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m         \u001b[0mq_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m         \u001b[0;31m# Greedy action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mq_values\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/policies.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, obs)\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0;34m:\u001b[0m\u001b[0;32mreturn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mThe\u001b[0m \u001b[0mestimated\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mValue\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0meach\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         \"\"\"\n\u001b[0;32m---> 59\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mq_net\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobservation\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36mextract_features\u001b[0;34m(self, obs)\u001b[0m\n\u001b[1;32m    117\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures_extractor\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"No features extractor was set\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0mpreprocessed_obs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocess_obs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnormalize_images\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize_images\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures_extractor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreprocessed_obs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/torch_layers.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, observations)\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobservations\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    115\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    422\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 423\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_conv_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    424\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    425\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36m_conv_forward\u001b[0;34m(self, input, weight)\u001b[0m\n\u001b[1;32m    418\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[1;32m    419\u001b[0m         return F.conv2d(input, weight, self.bias, self.stride,\n\u001b[0;32m--> 420\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    422\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "protos = sorted(glob('protos/*'))\n",
    "critics = sorted(glob('critics/*'))\n",
    "combined = sorted(glob('combined/*'))\n",
    "\n",
    "epochs = 25\n",
    "\n",
    "for testm in protos:\n",
    "#     print(testm)\n",
    "    selected = np.load(testm)\n",
    "#     print(selected)\n",
    "    env = DummyVecEnv([lambda: USPSGym(width=64, height=64, channels=1, dataset=(x_norm[selected, :], y_norm[selected]))])\n",
    "\n",
    "    # Grab the observation shape for generating evaluation frames.\n",
    "    width, height = env.observation_space.shape[0], env.observation_space.shape[1]\n",
    "\n",
    "    test_model = DQN(CnnPolicy, env, verbose=1)\n",
    "\n",
    "    test_model.learn(total_timesteps=len(x_norm) * epochs)\n",
    "    \n",
    "    test_model.save('models/norm_{}_{}.zip'.format(testm, epochs))\n",
    "\n",
    "    accuracy = eval(test_model, testx_norm, testy_norm)\n",
    "\n",
    "    error = 100 - accuracy\n",
    "    print(\"=============== m={} error={} ===============\".format(testm, error))\n",
    "#     break\n",
    "\n",
    "for testm in critics:\n",
    "#     print(testm)\n",
    "    selected = np.load(testm)\n",
    "#     print(selected)\n",
    "    env = DummyVecEnv([lambda: USPSGym(width=64, height=64, channels=1, dataset=(x_norm[selected, :], y_norm[selected]))])\n",
    "\n",
    "    # Grab the observation shape for generating evaluation frames.\n",
    "    width, height = env.observation_space.shape[0], env.observation_space.shape[1]\n",
    "\n",
    "    test_model = DQN(CnnPolicy, env, verbose=1)\n",
    "\n",
    "    test_model.learn(total_timesteps=len(x_norm) * epochs)\n",
    "\n",
    "    test_model.save('models/norm_{}_{}.zip'.format(testm, epochs))\n",
    "\n",
    "    accuracy = eval(test_model, testx_norm, testy_norm)\n",
    "\n",
    "    error = 100 - accuracy\n",
    "    print(\"=============== m={} error={} ===============\".format(testm, error))\n",
    "#     break\n",
    "\n",
    "for testm in combined:\n",
    "#     print(testm)\n",
    "    selected = np.load(testm)\n",
    "#     print(selected)\n",
    "    env = DummyVecEnv([lambda: USPSGym(width=64, height=64, channels=1, dataset=(x_norm[selected, :], y_norm[selected]))])\n",
    "\n",
    "    # Grab the observation shape for generating evaluation frames.\n",
    "    width, height = env.observation_space.shape[0], env.observation_space.shape[1]\n",
    "\n",
    "    test_model = DQN(CnnPolicy, env, verbose=1)\n",
    "\n",
    "    test_model.learn(total_timesteps=len(x_norm) * epochs)\n",
    "\n",
    "    test_model.save('models/norm_{}_{}.zip'.format(testm, epochs))\n",
    "\n",
    "    accuracy = eval(test_model, testx_norm, testy_norm)\n",
    "\n",
    "    error = 100 - accuracy\n",
    "    print(\"=============== m={} error={} ===============\".format(testm, error))\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "modified-forty",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1500, 2000, 3000, 4000, 4500, 6000, 9000, 12000]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ranges = [1500, 2000, 3000, 4500, 6000]\n",
    "ranges += [item*2 for item in ranges]\n",
    "ranges = sorted(list(set(ranges)))\n",
    "ranges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "noble-capitol",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}