.ipynb_checkpoints/multi-task-learning-example-PyTorch-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_tasks = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import time\n",
    "import pylab\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "import random\n",
    "\n",
    "def gen_data(N):\n",
    "    X = np.random.randn(N, 1)\n",
    "    w = [random.randint(-5, 5) for i in range(num_tasks)]\n",
    "    b = [random.randint(-10, 10) for i in range(num_tasks)]\n",
    "    sigma = [random.randint(1, 10) for i in range(num_tasks)]\n",
    "    #w = [2., 3., 4., -2., 5., 6, -4, 3, 1, 4]\n",
    "    #b = [8., 3., 5., 2., 4., 2, -2, 3, -3, 5]\n",
    "    #sigma = [1e1, 1e0, 5e0, 7e0, 3e0, 2e0, 4e0, 1e0, 5e0, 3e0]\n",
    "    Y = [0] * num_tasks\n",
    "    for i in range(num_tasks):\n",
    "        Y[i] = X.dot(w[i]) + b[i] + sigma[i] * np.random.randn(N, 1)\n",
    "\n",
    "    return X, Y\n",
    "\n",
    "def gen_val_data(N):\n",
    "    X = np.random.randn(N, 1)\n",
    "    w = [random.randint(-5, 5) for i in range(num_tasks)]\n",
    "    b = [random.randint(-10, 10) for i in range(num_tasks)]\n",
    "    sigma = [random.randint(1, 10) for i in range(num_tasks)]\n",
    "    Y = [0] * num_tasks\n",
    "    for i in range(num_tasks):\n",
    "        Y[i] = X.dot(w[i]) + b[i] + sigma[i] * np.random.randn(N, 1)\n",
    "\n",
    "    return X, Y\n",
    "\n",
    "\n",
    "class Data(Dataset):\n",
    "\n",
    "    def __init__(self, feature_num, X, Y):\n",
    "\n",
    "        self.feature_num = feature_num\n",
    "\n",
    "        self.X = torch.tensor(X, dtype=torch.float32)\n",
    "        self.Y = torch.zeros(num_tasks)\n",
    "        for i in range(num_tasks):\n",
    "            #self.Y[i] = torch.from_numpy(Y[i])\n",
    "            self.Y[i] = torch.tensor(Y[i], dtype=torch.float32)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.feature_num\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.X[idx,:], [self.Y[i][idx,:] for i in range(num_tasks)]\n",
    "\n",
    "class MultiTaskLossWrapper(nn.Module):\n",
    "    def __init__(self, task_num, model):\n",
    "        super(MultiTaskLossWrapper, self).__init__()\n",
    "        self.model = model\n",
    "        self.task_num = task_num\n",
    "        self.log_vars = nn.Parameter(torch.zeros((task_num)))\n",
    "\n",
    "    def forward(self, input, targets):\n",
    "\n",
    "        outputs = self.model(input)\n",
    "        loss = 0\n",
    "        precision = [0] * num_tasks\n",
    "        for i in range(num_tasks):\n",
    "            precision[i] = torch.exp(-self.log_vars[i])\n",
    "            loss += torch.sum(precision[i] * (targets[i] - outputs[i]) ** 2. + self.log_vars[i], -1)\n",
    "        \n",
    "        loss = torch.mean(loss)\n",
    "\n",
    "        return loss, self.log_vars.data.tolist()\n",
    "\n",
    "\n",
    "class MTLModel(torch.nn.Module):\n",
    "    def __init__(self, n_hidden, n_output):\n",
    "        super(MTLModel, self).__init__()\n",
    "        \n",
    "        self.shared_fc = nn.Sequential(nn.Linear(1, n_hidden), nn.ReLU())\n",
    "        self.nets = [0] * num_tasks\n",
    "        \n",
    "        for i in range(num_tasks):\n",
    "            self.nets[i] = nn.Sequential(nn.Linear(n_hidden, n_hidden), nn.ReLU(), nn.Linear(n_hidden, n_output))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        shared_out = self.shared_fc(x)\n",
    "        return [self.nets[i](shared_out) for i in range(num_tasks)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMkAAAByCAYAAAD5y/WzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2deWBU5d3vP8/sWUiGLSRhMYYdARdAcalaaVHLZm1Fpe0r79vW9vreFu17sS5tjfT1Smtvlbe3vdVrW5e6ISKLaKVFaQsChqAXBcKSiEoWA0Im20xmzsxz/zgzk1nOmS0hmYHz+SeZM2eeeWbm+Z7f8vye5wgpJQYGBvqYBroDBgbZjiESA4MkGCIxMEiCIRIDgyQYIjEwSIIhEgODJFhSOUkIcRRoB/yAIqWcKYQYArwEVABHgcVSylOnp5sGBgOHSGWeJCiSmVLKExHHfgmclFKuFELcAwyWUv44UTvDhg2TFRUVveuxgcFpoKam5oSUcrjWcylZEh0WAVcH/38a2AokFElFRQW7d+/uxVsaGJwehBAf6z2Xakwigc1CiBohxO3BYyOklE3B/5uBEb3oo4FB1pKqJblCStkghCgB/iqEqI18UkophRCafltQVLcDjBkzpledzQn2roYtK8B1DIpHwZyfwfTFA90rg16QkiWRUjYE/7YArwIXA58JIcoAgn9bdF77hJRyppRy5vDhmi7fmcPe1bDxh+D6FJDq340/VI8b5CxJRSKEKBBCDAr9D8wFPgQ2ALcFT7sNWH+6OpkzbFkBPnf0MZ9bPW6Qs6Tibo0AXhVChM5/Xkr5FyFENbBaCPFt4GPA8Clcx9I7bpATJBWJlLIeOF/j+OfAnNPRqZyleFTQ1dI4bpCzGDPufcmcn4E1L/qYNU89bpCzGCLpS6YvhgX/BcWjAaH+XfBf2Zvd2rsaHp0KVU71r5Fg0KQ3k4kGWkxfnL2iiCSUiQslGkKZOMio/+vea+CRNw/S2Oqm3JnH8msncsOFI/uwwwPHWSeSM/nHTItEmbg0RbLuvQbuXfsBbp8fgIZWN/eu/QDgjPhuzwiRpDLw173XQNWGfbS6feFjiX7MgRaTa+NGWh59DKWxEcxm8PuxlJdTctedFC9Y0Adv0HeZuEfePBgWSAi3z88jbx40RJINpHIViz0nEq0f87RdGVOcjXdt3EjTT3+G9HjUA361H0pjI00/VZMAvRZKH2biGlujLdLVn9awdP8bDHe3cnhzHwp7gMj5wD3RVSzROZHE/siptBkm1eD3tR/B2ttTmo1vefSxHoHEID0eWh59DIDqDY/TXDWOwAPFNFeNo3rD47qfEVTxX77yLc69ZxNVnV9DMTuiT8gwE1fu7MnoXf1pDcveX8MIdysmeoTt2rgx7XazhZy3JLEDXOu43jkhTEJw7j2bKHfm8cVJw2lIoU0g9eB372rY/UfUOtEIgjHAOv/lUa7d/21sQiTor9LURPWGx5la8xPyhBcElHKc4pqfUA3MWvi96PfesgLpOsYsOZQZvsU0cAVPdVxMh01hRcEr5Lube1VntvzaiWHLu3T/Gzj8vqjnQ8LOVWuS0nqSvmLmzJmyr0vlL1/5luagHunMY/s91yQ8J1VC7kOJuxVrZFzw6FRtl0WYQQbUgTd+LtQ8BVLbkkkEU/wvRlmupzc/REmX/vo1SyGMm9+oKaSPj5bg/WQMSlMTlqFFlExsoHh0W/j5LmnjHt932BC4AgCzEPyvxedn7EaGYidfYyMBYcIkA9oCF4LJB/Zn9B79gRCiRko5U+u5nHe3ll87kTyrOepYntXM8msnJjwnVSLdB0G0+yD1glzpJ+RSyd1/0BUIgIvCONfuT5Ovo9ts1TxfmCUlU09pDkTX0Tzc1SY12JcS5YSLxu0F1L4yAtdR1SXKF17utqwOf7Y//OXnTLj1SxyaMQnX0tK05ktCsZPSqArWrCcQwFJWllKb2UjuuFs6QW/oChjprjwz4iXGblgC6/0gzNwwYync+B9R53xx0nD+vPOTpG+r5z7UPfgg0+cJLNorBMLoDeaWvYNQusxY8v1sm3YH5RWtNMph/FJZDOdAkcWN/wNQusxqIxIs+X5KprdTXKFtFVv2DkL6Y697Aukz01RdDEBxhZuR4kRY/KHP5u8UwXNOUNyV2nxJotgpqgcOByV33Zn0vGwlN9ytWN8f1CBTazb7tR/B7j9ovPm3Yf6vww+TuWALTdu427Ka9tVWtIe6pHx2q+6A1cN1NI+m6uKowSzMAcpmuSiucNMtzQgENqHEiSmRQAAOvFim01cVS77C+IUtNFQX4aor1DwzdA7Fo+GuD9U+h9LRTU2Yi4sJANLlglSWfjudlN1/X9bHI4ncrdywJOlMfNU8pd1GzVMw/9ds+c0z2J76PU90nuJ4npOnplzP1tEzok5daNrGSuuT5Asvh/NLULq0viYRdXUGdcwInTEaOeBjB7L0m2jZO4jiCjd24cd1NI/6mqFInyl8rtJloXGnk6aaIspmtGmKxZLv1+krwTbMNFUX0VZXoCslpcvM4Q0lKF0Klo1zKLzqSlyvrgtbDH9rq277mqRgabKd3IhJ0pj4knr+v/Sz5TfPMOT3v2JY5ylMwAh3K8trXuC/vf8KoIpjm+2HrLL+jnzhBaBkejvCHNBpUh3cSbsftB7qANYenqp4IiyNL15MIdepcaeTo28NiWujsMxDXAYt8tXWAK11Bbp96OmL2k+lsZHWF15MyaXSIzJl7dq4kcPXzOHA5CkcvmZOzqSFc8OSpDjxte69BuZLExYRP6gDwoTtqd/HxRcmYP7RHeQN8/KdsW+GxRF+i+AVu3GnE63BpXSZcR3No7jCHbYisW6S3yc0YoV4DrxYFow/Eg9iELhb7DRVF9F+3vns6PgmHf5hOIaforJ9A6UtWi6tjLJM2sgkz2eG0tjIgUmT44712cToaSY3LEmKJeiPvHmQ5/zXxLnKUkK1nMqwzpOazZuAuft3xwkkNQSNu4o5tHYEB14s49DaETTu7LEaSpclODiTtwMiBYH0nF/bfhVbTv07HYESECY8jqHUTlxCc0mkay3D558ugVjKy5m8+kHM9vTi20grk83khkhSLEFvbHXzgPJvPOP/Eoo0ISUo0sQ/A+cxTdZiyddPxcou/bdXXaoEA0ia8HtV90j9G59hOh3UVS4iYLZHHQuY7dRVLszgvTPro2I2U/K12bgevQu/N/0kkNLUlPykASYr3S3t4sLEJeiujRt55q8PM7TzFKZ8Sft0O51jCvilspi7LavJF15Kprfruk0iX7tdKXvihf6muWQmdZUL6bYPwd59krH10a5Utz0+Lkl0vPfEiECAxa/Q8sc1SF8ByPSvubkwf5JVIln3XgMPbtzHqa7UKnVDuH57P03/Zy3DFPWx7BJ8Vl1MGS5WnvMkDnrcKGGRSAUihSLMAUqnuzSzU20fx7h5/URzyUxqJy4JW4ruoCsFhIVi7z5Jt2No3Gvt3SG3Ut+FSiZAbSLbkmHXUOkMTuSkSa7Mn2SNuxWqvI0USAjd4kKAvatp+ePLwYHfQyjzlC+8BDD1ZI2UyOBVIqx+yma5cEYE3pEkdbVOE3WVC5O6UmPrN2Dyd0edY/J3M7Z+Q/CRvkBqJy5RBSZEWIDRsUwy4jNvWsRJJ/glW8rLKfv5ioRBe7Zkw7LGkiSr1G1odXP5yrfi13dsWYHSqa31UM5/+PR2ju8t0pyNNltlOIPlOppHU01RONA22wL4vQNzHUnFlQpd+dO1CIkEmNya6KNlt0TEc/7CIsY88JOUslmxywUGMhuWNSJJVqkrIDxDHuWCuY5hyR+uO+GndFloqnaq++FrEDk/0birOMqvVoPwgbnxanJXSqW0ZXfaA/t0xTKJ7K0AzB1tfPyLX3EO8FnJLHasr6PjZDeFQ+xcumgsEy4pDZ8fW/ISdg9fy6dwx/a482PZVL+JVXtW0dzZTGlBKcsuWsa8ynkZfa6sEUm5M0+3TETL43X7/FRt2McNxaMomX4irtQjCr8AITXHeyjj1bJ3kE7gGXr3/nW5xtZviIpJINaVypxUBRhLZnFMDwKwnmhhz2OvcnBSPn6/+p12nOzmb88eoGrjfnYo8csFYuOzjpPdvP2cutOullA21W+i6p0qPH5VZE2dTVS9UwWQkVCyJibRq9R15ll1r+Wtbh/VY39AcYWbslkuLPkKuld+SdzMuTAHKCzzBMswkmWw+teilLbsZtLB57F7PgcpsXs+Z9LB53vlDoVIHsvE0zdxjErd6OvDAgkhFcnUzwNIVE/heL6z53wN91DxBtixvk6z/VV7VoUFEsLj97Bqz6q0+wpZZEm0qnlDcUeiYsQ7949ne94QiitOUlzh5tDaEUE3KZpQgWBoJtxsC+BXREplGgNFJq5Uqu2CdiyjZy304pj9k5dSV7mQoSc+4PNh01KyMnpuXVGg53f40+TruPP9Ndj9Pt3zO052ax5v7mxO63gyskYkAF88tofzNj+G0tSIpQBKak/B+cN4bMoPuOkd7bXXja1uWPILlPU/oLNO4PdpDHiTZPD0Toor3BRXuMOZLgKpGtLsEVFvXZ4QWgJMlHbWjVeCVqVx5FXhzFW3Yyj7J32TQ+NuQrEWxPVTz93L854E1AmrraNnIID7mv6ue37hEHvcMYDSglKaOuMnKUsL9GOYRGSNSOKyGR2E1zfM6nqAWx3f5QXP7LjXlTvzWOe/gG2+7/DdvWs044qA3YHpnPbw46YarUxX9pPK3EkqbYREhgyAMIUHcaKsl95ADRObPzdZUWzWcD/3T17KoXE3MeHIy1TWb+CgRrxVWbceRtwaPnZo2hXIH96OWH0IOqNz/BabiUsXjdXsyrKLlvHmE/fz9be6GdoGnxfBmmvsXHv7slS+ojiyRiRaC3h6Sshb+Gney6zzXx6VJg6tQHzkzYM0eC/jX7vWagZZJk83TjoANYuVWi1V36NlBQAOjvs6fmshABZfJxOOvKw56BO5PPsnL416rd57RSUDhOqWdjuGcmDSt5BCOy7rtg9hyoGn2D/pW2DKcMgIgWIrpHbiEs756HkmHXw+rn9lHbsBVSR5VjN3VJby9rP7UJToSUx7gYUrF0/UzW5dsS9AxRsBTEFvbHgbfO+NAKMuD0BlBl3PlkVXByZP0VnEI5l8SxMgWLdon2bMcu49m5DAU2/+JyPc8esdLAV+hi/owkk7tS+XDogVibUCAAR8gAlM0YNTBPyYlC781kLMvg4EAsVaEHwysesnAgpljdtoLrs06r1EQEEKE4gMPruUqkgm35bZ6zXai3MVTRKPxYbN6+NkwWC8S7/PkUOlKL54URYW+rntV1/Wbf7wNXM4ppTHiXCUpZHxb23RfE1OLLqylJWpa7Njj4eKEotHccOFIzVLU0Lp46emXM+P339eTfkGEeYAJdPa8H2az+H3RiD9AxNfaFkBTNrr2KXJjN+mrlMJ/U0VabLQOPLKuMEsM7UAAEKwf/LSzF+v0V7IVZRA2efVgBmHV622GNZ5iqZXt6KM/4ZmONjRkViox5SR1E68Nd4tPfgC4zPobtY45iV33YlwRO8DJcwBSqa3h8vi9coUVhQc4+nND7G85gWEKRBM9UpAYjJLuo5bObHTqrkqsIceK9ZcMpPts1fw1lX/m+2zV2SU5ozl9BUdanEaLgRCJLVi6RIw26mvXIg/IMAfnZ6vH3O97vvZPSdx/fZ+3Xbrx9+g6ZbWj78ho35mjSUJlRqoa6mD2a2pLorPH6YK5OM8zTKFrj17GBmxvFRd0dcz+ef3mlNM86qThs0ls+KC4/2Tl9JadC6Tjrys++rmkpnhbA6A2dfBxCNrwvFBv9LHg/l00m0fgknDy9a9qEjJ2PoNNFVXA1D87w/FneKxFmu+VO94MnolEiHEdcAqwAw8KaVc2Zv2is9xU7zgM3A1BXdE6Vkz0nLnnPjA3uOhdfXL4W1AI3qW5LEeQtstEoLGkVfhbPtIM6BuLpmpBr4RLo3fNoj9k5dyYMItSLO9/wZuooX2WYi9+yQBAWYZf1wrm2b2dVDashuJoOVPazVFUjjEoTmHUjjEEXcsFTJ2t4QQZuC3wPXAFOBWIcSUTNtLdlNOrXgF0BBIchK5U4nmA6IXM/W0tX/Sv2j7/EIgLXl9E+ymSg4JxOTvZszRDWy+EDwxX9+YoxuQMr4qYOKRNeHHSod20slywWB8MRUSPiSWCwZn1M/eWJKLgSPB28UhhHgRWARktk1fxI4oUWvEX32AwgUH9F8X3HE9GVHzA2qHgfi5hkTzAbECqh13U9QkmkGKSInZ10FZwxrWXlzD9vMsHBrlZ8lWGZ7XWHtxDZ+VmLmybh62gPbEqaVQ+3v/XX0zg/J8XOmxUCQFbULyD4dCe30z3yD963hvRDISiNyd4RhwSexJKd/HPbjzSey+VEoHtL7wou7LnItvitryRgs1zrg13o0KElkmPrZ+g5rJ0Rj4Fl8n22evoNs+BIuvU40/DIGkjRQSxVbIgYmL+KzEDNSw/Twz28+LPbOGI8NruHyfn++9LnFEzCcGLJKSf/2aZvuNrW4W5G3jXwatplycoFEOo1lZzMbWKzLq72n3A1K+j3tw5xPtXQh1W6fsq5Mp+/kKLOXlIASW8nKct94S9biucoGuQEKErERpy27KG/4eN2cjAn4Ua0G4wE+xFRoCyRCBCYFgkHcIV9XfwvQTF2MJRH/fjoDk5qEzKCsoY/t5Zh7/iuB4EQSA40Xw+FdsbLv+Ms32byt8l5XWJxllOoFJwCjTCVZan+S2wncz6m9vLEkDMDri8ajgscyY8zPY+MO01pNb8v2wZQXFd32YcCHO5u9pTyBFEioTby6ZyefDpqkHw0KRwYk4QxR9jTVgY+5H17Mk8DqrBjtptpgpVfwsO9XKvNbdcNeHzF0zl+3nNcVYGkntnlWape93W18iX4ne+SZfeLnb+hLwYNp97I1IqoHxQohzUcVxC7Ak49aCWSzLqw+gdGidEL2mIzyH4jqetOmCfEmnW3+Ah8rE1SD8mxqTfIY4EtLLjFqH4uTS4zP4xkffpCMwjELTCcYX/hnytwHQpFO9q3c8353e8WRkLBIppSKE+O/Am6gp4D9KKfdl2h4A0xdT8kD0fAioGwYUV3roOKrE74tbPDqqia1/WEHBb1+ioEudnPIPyuOCb/8nOz+wR69hCFoJi68TiQzOKMv+zUQZAOAwdfF22x0oqCnajkAJb7fdAflDmQAIxYm0xN+KQijOuGNAn97FC3o5TyKlfB14vTdtxFK8YAHvtbyH9YnVOF1+WovN+G7/KpNmTdLeNDtig7qtf1iB8uxh/t/0qqianZLfLGf2Dx7h/YM2OrtEuPrV4utEsdgjLIdhMdLFZ/IiAt1YSK98JoSZbrqkDRO2qOMKDnZ0fJPD9Zvw042IMVYyYMXz2VztRoOue6Kxkg5Zd9ncVL+J5faNfP8OwS33Wvj+HYLl9o1sKixIukFd+8t1HB4fv3quZehFmJ7/OX+tWEtAeNWCwlDwrVM/ZZACMoAivJgpRKa5clMiKTS1cHXR7xBS+zfo6DBR9U4VJktXWCBSQkDJw9N0IyUm7cA91c0MUyVrylJCJFp6OX70M+w4/ri6eYDXzqXusUyIOO+z0vm66yFG7PwZ0+quwUTiLJdBOgjy/IVpv8rk72ZI2ZPcbPobxwLDaGuXFGts79rlaIsbC0KADNixumey/MaJca8JMz3xZobpkFUiObSrmWv+cTuF3sFIAghMdNhOsWvMa3Ac3v5HLYpXjTW0NgNItAvIiSIo9GY242qgQ6rBupRRC7yGN21g0ci/0SVt/FJZzCGHwnVuK9YId9diM/HOqPWazZmsrTx047R+u/111ojk0K5m3n6ulkFedaAL1FRwKJceMCsovuhK0dBmAOHFN0J/V5NNl81icOBUuH2D3iGRiFRiuICPKbV/Ds+Ueyywei7MCKh39doQuALsasXENV4bBX7CWwy98mkjdMY3WVZY1q/3h88akexYXxe2ErFYAzYI6PitUYVsOiGWEAwOLOSo80OmtnxB88dN+Uc3QCLx48MSE2wDBAiEv0ePuZPjjjVM8+wmQM8y2qFz/4Mvvz0cd6CnnOjjQphw48Sowb9s+DJ+uu0BfBE1XFZhZ9lFmS3DzZSsEUnHSQ/JtzeLJ3IzgMIhdt0dNAq9g6lonaorBEMg+kgkHnMnDn8BHbZTHHV+yJTjl8XtsqSgsHXc8xwZXhN1/PUL1WFWVlAW3iTu/MFam6JHWwef6wI8TTcihryBsLYifU48J6/H57rgtH7eWLJGJF2ONvI9+vX+whHA5/VjibAosZsBXLpoLH/9k3Z9paVIUthmxCSJ0LOmHbZTPDejZ6b6GzUPYJbxQ8dn8cQJJIRAsPnrm8OP9VaZRvLImwfpaj0fTp0fd7w/3a2sSQG/M2o9PpP2TXSERbJ1zEtsrXyBdttJJJIO+ymccz1RmwFMuKSUqVeWx73eYjMx+YJRhrVIgtb34zN51cRJBHoJEIdSoNt2Jtv56G19m2xL3L4mayxJZ0Ujf+dFLvlkflR2y+1oY//YrRwo2gkQdaUq85RxM9dFtdMxu476un1UNFyoFtIJmDS7lKMfft6vn+dMIICfv1e+GP7OQ9UnHTbtBEiHLX5WHMBhdmQUR+htfVvu7N/bYWSNSJZdtIwqT1WUCBxmB1WXVfHsP9/WfE3sjnyb6jfx9Lp1XNb0NUzB7BgS9r3TgFRStyJ9HcTnalJAYIr6PQSCPLODXaNf46qPblETKkG0LI6U4LSVcO/sH2W0B+/yaydy79oPNLeR6k+yxt2aVzmPqsuqKCsoQyAoKyij6rIq5lXO0zXVscdX7VnFRUevjfrxgLQEAmd+EJ/q7HicZRASp8PJ+5/fxF/s4BIBJJI2s4et574cJSgZsOJpvBn5yf0Z7+Z+w4UjefjGaYx05iGAkc48Hu7H+ZEQWWNJQBWK1he67KJlUbuEQ7wJj5yIzCZC6VIz1qwQn0TiM3VjCyRe761lGUC13uXOPGpb3dTae67wFt+52L2Hwlmo7uPXorRdSCO9ix9SCfBPN1klEj3mVc6jY7+ZT/7mJs9ThNvRxpgv5TGvUo1HYiciB5rQlVoGtzXSmk9ItZ10hOUXCl6TB4e/AI+lE6viwBLxE0sknw6q5dCIaq6qj3aXIl8bqnLQylSVFpRyh4YbpLRdiNJ2Ydz5/R0/nA5yQiSHdjXTutlBvlf9UfM9xbRuNnFoeDMTLinlrVc+THhHqnQHW7LzU2lvy7hnueST+b0Sbqp9lkjNgT3u+IxwIkTrebV/g2m3n2LX6HhRxC4TkQFrcJ4j/g4AX5w0nFdqGgY8fjgdZM02p4l4+r7tmpOE5qIAo7/fzUe/dCQcUAH8BJBRV1U9fCYvtcN2MvH4xVhlxDahEe2nIhKfyYsl0D8uVrvtZNQ8RqoIBHtv28u0p6dpPi8lSJ8z7ELldy6g+s67ddvTvmvywLpKqZIT25wmQm8WXWkTPLzrYebb7kx4xTZhptvSgdvkpdA7GJ+pG0vAhkAgkSjCi1Xae4opgUknZiM0KlMhtSu8NWAjgD9cg5YuqVo/vdhBi1jLcnD8PwF1JlzrVgXS56Sz7h5AtQr336gtphDZED+cDrImu5UIvftQdNhO4fK6OOr8MGnGxqEU8NyMB9ky7lkEJkzBzQhMmBDCxJZxz/LWlU/QULqfSz6ZH5chywSBSXeCNBKJxG3pwG3uQCJpt53EJ7QvDBJJAH/4vMh5jESMOz6Dq+pvYZB3SHgThlkHF3JoVzPLLlqGwxwdyFuFnfzOBQOaVcoWcsKSXLpoLK8//b5mXn7c8RnqVT/JVbfTru42ryUAa8DGDSe/y23/43KmPz09pQxZKlf6kGWKnCA1aViW2LIPUAf11UeWRLmIerVRqaD1uaUi2LG+jtv+p5pR7KsbcZ5p5IRIJlxSysPvvsHEw1+IC0K/tefBpFd9n8nLztEbWfmFlXy0Qzv1qRZYqtkbvRnlyPZqh+2konUqhd7BmpmkkIiPDK8JD+rQ1TzZJBz0VBYkCry1MAkTUso4y6on/JArq5d+N8gRkQB8fd5czbmSgm7tosjQIAkNrs6KRuZVzuMRx6uahZRdjjZAnZN58ujLcYM5sr13z9nE4WG72c4r4eeTZZJAY+DrZJUiz0/HaoQqFOZVzmPumrlRcYae8PVcWYMeckYkoatcrEtwYr/25siRLozD7KDqoipALaTUupqrq+C+yrzKeaybti6qjiw06GdcMY6fzP4J059eEfd+qQ7o0HmhsvFN79aBdviRlDxzHnaLHVe3K85Fip2A3TXmNa6uvwVLxOdOdEs1gx5yRiSg7RIcWtSsG69A9BoGiC+kjLQ0IT5u/5im4U1xg77zmHpOsb2Y1u74O2qlSqhaIPR5NtVvYuW7K5O2GftZEhF7UemsaKR0EnjeUdfchFb/6d1SzaCHnBKJFonilbKCsqg1DJCgkDJoaSD5LY7TnVty2p3kWfJ0g+JIsdy37T4CMn6FptZnSYZmnDE/rSYMOANEAvrxilZ5tp7bFjmYkt3iuM3bllb/7rn4nrSu/ql+FoP+4YwQSezAL7IVIYTg3n/ey6o9q3Sv3HokK6jUE5EWN0+8Oa2sUSoiNuhfcqIsJR021W/SHOChrE867egNVL33WDRuEf849g9jcOcgicpSzjiRxKY+Q2Ti0ycikYgMco+cr91Kh2RBd19hTL6dPeRE7VY6pLqK0cAgVc44kWgV6xnZIYPecMa5W0Z2yKCvOeNEAka8YNC3nHHuloFBX9OvKWAhxHHg4357Q22GAScGuA+Zkqt9z4V+nyOl1Lw9dL+KJBsQQuzWy4dnO7na91ztdwjD3TIwSIIhEgODJJyNInlioDvQC3K177nab+AsjEkMDNLlbLQkBgZpcVaKRAjxiBCiVgixVwjxqhDCOdB9SoQQ4johxEEhxBEhxD0D3Z9UEUKMFkK8LYTYL4TYJ4TIydqgs9LdEkLMBd6SUipCiF8ASCl/PMDd0kQIYQYOAV8GjgHVwK1SSu373mURQogyoExKuUcIMQioAW7Ihb5HclZaEinlZimlEny4Exg1kJh2FNsAAAE7SURBVP1JwsXAESllvZTSC7wILBrgPqWElLJJSrkn+H87cADIuW0gz0qRxPBvwBsD3YkEjAQ+jXh8jBwcaEKICuBCYNfA9iR9zsgCRwAhxN8ArUUk90sp1wfPuR9QgOf6s29nG0KIQuAV4E4pZXq7aGQBZ6xIpJRfSvS8EGIp6gY7c2R2B2YNwOiIx6OCx3ICIYQVVSDPSSnXDnR/MuFsDdyvA34NXCWlPD7Q/UmEEMKCGrjPQRVHNbBESrlvQDuWAkIIATwNnJRS3jnQ/cmUs1UkRwA7ELpv9U4p5fcHsEsJEUJ8BXgMMAN/lFI+NMBdSgkhxBXAP4EPgNCOe/dJKV8fuF6lz1kpEgODdDCyWwYGSTBEYmCQBEMkBgZJMERiYJAEQyQGBkkwRGJgkARDJAYGSTBEYmCQhP8PJ8SYkCvBnz8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x108 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "seed = 3\n",
    "np.random.seed(seed)\n",
    "\n",
    "feature_num = 600\n",
    "val_feature_num = 200\n",
    "nb_epoch = 500\n",
    "batch_size = 20\n",
    "hidden_dim = 1024\n",
    "lr = 0.0001\n",
    "\n",
    "X, Y = gen_data(feature_num)\n",
    "X_val, Y_val = gen_val_data(val_feature_num)\n",
    "pylab.figure(figsize=(3, 1.5))\n",
    "for i in range(num_tasks):\n",
    "    pylab.scatter(X[:, 0], Y[i][:, 0])\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from torch.utils.tensorboard import SummaryWriter\n",
    "from tensorboardX import SummaryWriter\n",
    "writer = SummaryWriter()\n",
    "writer.add_hparams({'feature_num': feature_num,\n",
    "                    'val_feature_num': val_feature_num,\n",
    "                    'nb_epoch': nb_epoch,\n",
    "                    'batch_size': batch_size,\n",
    "                    'hidden_dim': hidden_dim,\n",
    "                    'lr': lr, \n",
    "                    'seed': seed, \n",
    "                    'num_tasks': num_tasks}, {})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "expand(torch.FloatTensor{[600, 1]}, size=[]): the number of sizes provided (0) must be greater or equal to the number of dimensions in the tensor (2)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-705366709dcd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0muse_cuda\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mval_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_feature_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtrain_data_loader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop_last\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-e0517bb8a088>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, feature_num, X, Y)\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_tasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0;31m#self.Y[i] = torch.from_numpy(Y[i])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: expand(torch.FloatTensor{[600, 1]}, size=[]): the number of sizes provided (0) must be greater or equal to the number of dimensions in the tensor (2)"
     ]
    }
   ],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "\n",
    "train_data = Data(feature_num, X, Y)\n",
    "val_data = Data(val_feature_num, X_val, Y_val)\n",
    "train_data_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "val_data_loader = DataLoader(val_data, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "\n",
    "model = MTLModel(hidden_dim, 1)\n",
    "if use_cuda:\n",
    "    model = model.cuda()\n",
    "\n",
    "mtl = MultiTaskLossWrapper(num_tasks, model)\n",
    "\n",
    "# https://github.com/keras-team/keras/blob/master/keras/optimizers.py\n",
    "# k.epsilon() = keras.backend.epsilon()\n",
    "optimizer = torch.optim.Adam(mtl.parameters(), lr=lr, eps=1e-07)\n",
    "\n",
    "loss_list = []\n",
    "val_loss_list = []\n",
    "for t in range(nb_epoch):\n",
    "    start = time.time()\n",
    "    cumulative_loss = 0\n",
    "    cumulative_val_loss = 0\n",
    "    \n",
    "    for X, Y in train_data_loader:\n",
    "        if use_cuda:\n",
    "            X, Y = torch.FloatTensor(X).cuda(), torch.FloatTensor(Y).cuda()\n",
    "        \n",
    "        loss, log_vars = mtl(X, Y)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        cumulative_loss += loss.item()\n",
    "        \n",
    "    for X_val, Y_val in val_data_loader:\n",
    "        if use_cuda:\n",
    "            X_val, Y_val = torch.cat(X_val, dim=0).cuda(), torch.cat(Y_val, dim=0).cuda()\n",
    "        val_loss, _ = mtl(X_val, Y_val)\n",
    "        cumulative_val_loss += val_loss.item()\n",
    "    \n",
    "    loss_list.append(cumulative_loss/batch_size)\n",
    "    val_loss_list.append(cumulative_val_loss/batch_size)\n",
    "    \n",
    "    end = time.time()\n",
    "    epoch_time = end - start\n",
    "    writer.add_scalar('Time/train_epoch_time', epoch_time, t)\n",
    "    writer.add_scalar('Loss/train', cumulative_loss/batch_size, t)\n",
    "    writer.add_scalar('Loss/val', cumulative_val_loss/batch_size, t)\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pylab' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-5ad44b21c7a3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpylab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mpylab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavefig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'train_loss.png'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mpylab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pylab' is not defined"
     ]
    }
   ],
   "source": [
    "pylab.plot(loss_list)\n",
    "pylab.savefig('train_loss.png')\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHLVJREFUeJzt3XuUHOV55/HvU93TPVfNSJrRIEYSAiOICRDAE6w12MEmIYAdC+/aLDn2Ijva1Vkv2bXjzXFIvCdxNjm7dmzHa8cb2yT4AI4vEDsE4nUWYYwv2YTLcDEXgdAgBEjWZSSN7ppLdz/7R709ao3mKk13T1f/PufMmeq3qruerq75zdtvVVebuyMiIskVVbsAEREpLwW9iEjCKehFRBJOQS8iknAKehGRhFPQi4gknIJeRCThFPQiIgmnoBcRSbh0tQsA6Ozs9JUrV1a7DBGRmvLEE0/scfeu6ZabF0G/cuVK+vr6ql2GiEhNMbNXZ7Kchm5ERBJOQS8iknAKehGRhFPQi4gknIJeRCThFPQiIgmnoBcRSbiaDvrHt+7jsw9sIpcvVLsUEZF5q6aD/qnXBvnSw/0M5RT0IiKTqemgz6Ti8kcU9CIik6rtoE+nAAW9iMhUajzo1aMXEZlOMoI+n69yJSIi81dtB30Yox9Wj15EZFI1HfRZDd2IiEyrpoNeY/QiItNLRtDrA1MiIpOq6aBv0Hn0IiLTqumg1wemRESmV9tBr6EbEZFp1XTQF8+60emVIiKTq+mg11k3IiLTq+2g1xi9iMi0ajvoNUYvIjKtRAT9qHr0IiKTqumgT0eGmXr0IiJTqemgNzMyqUhj9CIiU5hR0JvZVjN71syeNrO+0LbIzB40s83h98LQbmb2RTPrN7NnzOyycj6BTDrS6ZUiIlOYTY/+7e5+ibv3htu3Ag+5+yrgoXAb4DpgVfhZD3x5roqdSDYdaehGRGQKpzN0swa4M0zfCdxQ0n6Xxx4BOsxs6WmsZ0oauhERmdpMg96BDWb2hJmtD23d7r4jTO8EusN0D/B6yX23hbayyKQV9CIiU0nPcLkr3X27mS0BHjSzF0tnurubmc9mxeEfxnqAFStWzOauJ1DQi4hMbUY9enffHn7vBu4FLgd2FYdkwu/dYfHtwPKSuy8LbeMf8zZ373X33q6urlN+AhmN0YuITGnaoDezFjNrK04D1wDPAfcDa8Nia4H7wvT9wM3h7JvVwIGSIZ45pzF6EZGpzWTophu418yKy3/T3f+vmT0O3GNm64BXgRvD8t8Hrgf6gaPAh+a86hIauhERmdq0Qe/uW4BfmqB9L3D1BO0O3DIn1c1AJp3iwLHRSq1ORKTm1PQnY0FDNyIi06n5oM+mI0Zy+WqXISIyb9V80OusGxGRqdV+0GvoRkRkSrUf9DrrRkRkSgp6EZGEq/mgz+oyxSIiU6r5oG9sSJErOKM6ICsiMqEEBH38FIZGdYqliMhEEhD0KQCGRtWjFxGZSIKCXj16EZGJJCboh/XpWBGRCdV+0KeLY/QauhERmUjtB72GbkREppSYoD+moBcRmVACgl5DNyIiU6n5oG/S0I2IyJRqPug1Ri8iMrWaD/pscehG17sREZlQzQf92Hn06tGLiEyo9oM+Hc66GVHQi4hMpOaDviFlRAZD+mSsiMiEaj7ozYymhpROrxQRmUTNBz3E4/Q660ZEZGIJCnr16EVEJpKIoM82ROrRi4hMIhFB35jW0I2IyGQSEfRNmZTOuhERmUQigr6xIdIYvYjIJJIR9Bq6ERGZVDKCXqdXiohMKkFBr6EbEZGJzDjozSxlZk+Z2ffC7bPN7FEz6zezu80sE9qz4XZ/mL+yPKUf15xJcXQkV+7ViIjUpNn06D8CvFBy+9PA5939XGAQWBfa1wGDof3zYbmyas6mOKKLmomITGhGQW9my4B3An8dbhvwDuA7YZE7gRvC9JpwmzD/6rB82bRk0ozkCuTyGr4RERlvpj36/wV8HCgm6WJgv7sXx0u2AT1hugd4HSDMPxCWL5vmTHyp4qM6ICsicpJpg97M3gXsdvcn5nLFZrbezPrMrG9gYOC0Hqs5kwbg6LCCXkRkvJn06K8A3m1mW4FvEw/ZfAHoMLN0WGYZsD1MbweWA4T57cDe8Q/q7re5e6+793Z1dZ3Wk2jJxj36IzogKyJykmmD3t1/392XuftK4Cbgh+7+fuBh4L1hsbXAfWH6/nCbMP+H7u5zWvU4xR69vmVKRORkp3Me/e8BHzOzfuIx+NtD++3A4tD+MeDW0ytxesUx+iPD6tGLiIyXnn6R49z9R8CPwvQW4PIJlhkC3jcHtc3Y2MFY9ehFRE6SiE/GtmTDwVgFvYjISRIR9E0NOhgrIjKZRAR9sUevg7EiIidLRNCPHYxVj15E5CSJCPpsOiIyfWBKRGQiiQh6M6Mlk9bBWBGRCSQi6CG+gqUuVSwicrLkBH0mrUsVi4hMIEFBn+KYevQiIidJTNC3ZNIc0cFYEZGTJCbo42+ZUo9eRGS8xAR9azbN4SEFvYjIeIkJ+rbGBg4q6EVETpKYoF/QmObQ0Gi1yxARmXcSE/St2TTDuQIjOX1BuIhIqcQEfVtjfGGzw/ryERGREyQo6BsANHwjIjJOgoI+7tEf0gFZEZETJCboWxX0IiITSkzQL9DQjYjIhBIT9Bq6ERGZWIKCXj16EZGJJCboW7Pq0YuITCQxQZ9JR2TTkc6jFxEZJzFBD7rejYjIRBIV9LrejYjIyRIV9K2NaY3Ri4iMk6igb2tMa4xeRGScZAV9tkFDNyIi4yQq6NubGjhwTEEvIlIqUUHf0dLA4NFR3L3apYiIzBvJCvqmDCO5AsdG89UuRURk3pg26M2s0cweM7OfmdnzZvbHof1sM3vUzPrN7G4zy4T2bLjdH+avLO9TOG5hc3wZhP1HNXwjIlI0kx79MPAOd/8l4BLgWjNbDXwa+Ly7nwsMAuvC8uuAwdD++bBcRXSEoB88OlKpVYqIzHvTBr3HDoebDeHHgXcA3wntdwI3hOk14TZh/tVmZnNW8RQ6mjMAHFCPXkRkzIzG6M0sZWZPA7uBB4GXgf3uXjxpfRvQE6Z7gNcBwvwDwOK5LHoyx3v0CnoRkaIZBb275939EmAZcDnwC6e7YjNbb2Z9ZtY3MDBwug8HwMLQo99/TEM3IiJFszrrxt33Aw8D/wroMLN0mLUM2B6mtwPLAcL8dmDvBI91m7v3untvV1fXKZZ/ovYmHYwVERlvJmfddJlZR5huAn4NeIE48N8bFlsL3Bem7w+3CfN/6BU6sb2xIUVTQ4r9OhgrIjImPf0iLAXuNLMU8T+Ge9z9e2a2Efi2mf0p8BRwe1j+duDrZtYP7ANuKkPdk+pobtAYvYhIiWmD3t2fAS6doH0L8Xj9+PYh4H1zUt0p6GjOaOhGRKREoj4ZC9DR1KChGxGREokL+oUtDezXhc1ERMYkLujbmzIMHlGPXkSkKHFB39maYfDoCPmCrmApIgKJDPosBYd96tWLiAAJDPqutiwAew4PV7kSEZH5IXFB39mqoBcRKZXAoI+vd6OgFxGJJS/oi0M3hzRGLyICCQz6tmyaTDpiQD16EREggUFvZnS1ZtlzSEEvIgIJDHqIx+nVoxcRiSU06LPsOawxehERSHTQq0cvIgIJDfqutiz7joxQ0GUQRESSGfSdrRnyBWdQlysWEUlm0C9Z0AjAroMavhERSWTQn9EeB/3Og8eqXImISPUlMujPbG8C4Of7h6pciYhI9SUy6LvasqQiY8cB9ehFRBIZ9KnI6G7LsuOAevQiIokMeoClHU3s0NCNiEhyg/6M9kYN3YiIkOCgP7O9kR0HhnDXh6ZEpL4lNujPaG9iOFdg8OhotUsREamqxAb9meFc+p/v1/CNiNS3xAb90o74XHqdeSMi9S6xQd8Tgn7b4NEqVyIiUl2JDfrO1gzNmRSv7lXQi0h9S2zQmxkrFjXz2j4FvYjUt8QGPcBZi5t5de+RapchIlJViQ76FYuaeX3wmL6ARETq2rRBb2bLzexhM9toZs+b2UdC+yIze9DMNoffC0O7mdkXzazfzJ4xs8vK/SQms2JxCyO5ArsO6cwbEalfM+nR54D/6u4XAKuBW8zsAuBW4CF3XwU8FG4DXAesCj/rgS/PedUzdNaiZgAdkBWRujZt0Lv7Dnd/MkwfAl4AeoA1wJ1hsTuBG8L0GuAujz0CdJjZ0jmvfAbOWhwH/WsKehGpY7MaozezlcClwKNAt7vvCLN2At1hugd4veRu20JbxZ3Z0UQqMl7dpwOyIlK/Zhz0ZtYKfBf4qLsfLJ3n8ZXDZnXE08zWm1mfmfUNDAzM5q4z1pCKOGtRMy/vVtCLSP2aUdCbWQNxyH/D3f8uNO8qDsmE37tD+3Zgecndl4W2E7j7be7e6+69XV1dp1r/tM5d0srm3YfK9vgiIvPdTM66MeB24AV3//OSWfcDa8P0WuC+kvabw9k3q4EDJUM8FXdedxtb9x5lOJevVgkiIlWVnsEyVwD/DnjWzJ4ObX8AfAq4x8zWAa8CN4Z53weuB/qBo8CH5rTiWVrV3Uq+4Gzdc5Tzz2irZikiIlUxbdC7+z8BNsnsqydY3oFbTrOuObNqSRzuL+06pKAXkbqU6E/GApzT1UJksHn34WqXIiJSFYkP+saGFCsWNdOvA7IiUqcSH/QAq7rbeGmXevQiUp/qIujP727jlT1HGBrVmTciUn/qIugv7GknX3A27jg4/cIiIglTF0F/0bJ2AJ7bfqDKlYiIVF5dBP2Z7Y0sbsnw7DYFvYjUn7oIejPjwp52nlWPXkTqUF0EPcDFy9rZvPuwDsiKSN2pm6AvHpB9/uc6ICsi9aVugv6S5R0APPXaYJUrERGprLoJ+u4FjaxY1Mxjr+yrdikiIhVVN0EP8MsrF9H36iDxdddEROpDXQX95WcvZN+REV4e0OUQRKR+1FXQ//LKRQA89orG6UWkftRV0J/d2UJna5ZHX9lb7VJERCqmroLezLji3MX80+Y9FAoapxeR+lBXQQ/wK+d1sffIiM6nF5G6UXdB/9ZVXQD8+KXdVa5ERKQy6i7ou9qyXNTTzo82DVS7FBGRiqi7oId4+ObJ1wYZPDJS7VJERMquLoP+ml/spuCwYePOapciIlJ2dRn0F/W0s3xRE//nWQW9iCRfXQa9mXH9RUv55/497D+q4RsRSba6DHqAd160lFzB2fD8rmqXIiJSVnUb9Bf1tHPW4mbufWp7tUsRESmrug16M+PG3uX8y5a9bN1zpNrliIiUTd0GPcB737SMVGTc0/d6tUsRESmbug767gWNvP38JdzTt43hnL5LVkSSqa6DHmDtW85iz+Fh7nv659UuRUSkLNLVLqDarjy3kwuWLuC2n2zhvZctI4qs2iXVtHzBSc1yGxa/8av0i7+Kk6P5Ag2piNF8gXRkpFMRuXwBgJEwD6Dgjnu8/mw6YjTvFNxJp4x0FN//8HCOlkyaVGQ4Pra+XMFJmTGSLxBZ/Bj5glNwSEfGaKFAcyZNwZ1CwRnNO2ZxvZFBKjIKztj8gjP2+F7yHIvrM4OWTHrs+REeK36OEdl0hJdul7BtnPgBS2/72O3jjz9+HkDKjEw6GtueHuYYhlm8DQrhtcumI1KRcWw0Ty7vRJFhEKahIRVR8HgbjeZ9rN7IYGi0QC5foCmTwiy+n9nx9RT3jEPDOdJhu8XbPk9kRkMqYiRfGKuh4HBsJH63HRlEZqQiI1dw0pHhDqmUMZIrkDI7YXool6epIXXS/nZ0JE82HT+HhlTEcK5AJhURRTCcKzCSK9CSSTNaiPe54raxsG6AfKGAmZGObGx/ibfr8X0x3voGOJFZvJ50hHt8uxBe8+4FjSxqyczwr+XU1H3Qmxnr33YOH737aR7etJur39hd7ZImlcsX6B84zJaBI7yy5wiDR0bYe2SEw8M5UmbkCgVG8s5orkCuUBj7Qy3uiFgcXEOjBfKFOASPjeTjnTPsqLmC05CKd8rIjHwhXjaTjhgaLWBAJh0vN5IrnBCYBfdw/4hMKgpBUIh3/JLncTrf5BgZ6ArTkiR/esOFfGD1WWVdx7RBb2ZfA94F7Hb3C0PbIuBuYCWwFbjR3QfNzIAvANcDR4EPuvuT5Sl97rzz4qV85oFNfOGhzbz9/CXzqlc/mi+w4fldPPD8Tn780gAHjo2OzWtsiOhszdKaTY/1TuKfuGfUmk2H5VKhFxsHejr0yCDuCbY3NYz1gA0jsjjMi73zYg8qE3rPuUJhrIcYWdhWY78M97inl4ognYpIh+15wla1E9us5P7jF8sXnGxDRC4f/3NpSEXk3WkIz8mAKDIii3uNI7kCqcjGemO58NzaGtMMjebD49oJ63WHbHjOmXSEwdjzTkXG0Gjc4yw+33QqrjRf8LEedi70/It1lPZmi0/WiP8hHhrKMTSap6M5Qy5foODQlEkdry/UWNoLZpIe8tg6rOR+xfYwL1dwhnP5sZ5n3M9kbD9IR4aZUSg4w7nj/9wz6QjCu5V0Kgr/+ON/uITHyBd718S95aaGVLhd+o7jxHcZUXgXU3yXVuzlFve5fHjHUHxts+l438uH+ox434K4A1TcX4dG87Rk08ffIaVOHp0+Nhr36HNhH01F4V1hyXM+eGyUpkxqbH8r3U889NDjdTt597F9Z+y1NyOXL5DLO42ZFIWCj71TsdCBSkURDZHxi2e2n1TjXJtJj/4O4EvAXSVttwIPufunzOzWcPv3gOuAVeHnzcCXw+95rSEV8Tu/dh6/+7c/4/vP7eBdF59Z7ZIA+OnmAf74HzbSv/swna0ZfvWN3Vy5ajGrlrSxtD1+u1cMLBGRyUwb9O7+EzNbOa55DXBVmL4T+BFx0K8B7vJ4cPERM+sws6XuvmOuCi6X91zaw1//dAufeWAT11xwRtyTqZKdB4b4w/ueY8PGXZy1uJmvfOBNXHNB97x6pyEiteNU06y7JLx3AsWB7R6g9KT0baFt3ktFxq3X/QKv7j3KV378ctXq+OnmAa7/4k/56eY9fPza89nwO2/j2gvPUMiLyCk77W5r6L3P+vCYma03sz4z6xsYmB9fAnLV+Ut458VL+dIP+3l54HBF1+3ufPXHL3Pz1x6jszXD9/7Llfynq84lmz75rAERkdk41aDfZWZLAcLv4vfybQeWlyy3LLSdxN1vc/ded+/t6uo6xTLm3h/9xgU0NkR8/DvPxKe+VcBovsAf3Pss//MfX+T6i5by97dcwRu6WiuybhFJvlMN+vuBtWF6LXBfSfvNFlsNHKiF8flSS9oa+ZMbLuSJVwf57AObyr6+Q0Oj/NYdj/Otx17nlre/gb+46VKaM3V/1quIzKGZnF75LeIDr51mtg34I+BTwD1mtg54FbgxLP594lMr+4lPr/xQGWouuzWX9PD41n189SdbuHRFB9deuLQs69l9cIgP3fE4m3Ye4s/eezE39i6f/k4iIrNkfjqfXpkjvb293tfXV+0yTjCcy3PjVx/hxR0H+fq6N3P52Yvm9PGf2baf9Xc9wcGhUf7y/Zdx1flL5vTxRST5zOwJd++dbrm6v9bNZLLpFF9b20vPwibW3fE4T702OGeP/Q8/+znv+8q/kIqM7374LQp5ESkrBf0UFrdm+fq6N7OwJcNv/tUj/GDj6X0bVaHgfG7DJv7zt57i4mXt3PfbV/DGpQvmqFoRkYkp6KfR09HEdz/8Fs7rbuM/fL2Pzz6w6ZTOxtm+/xgfvONx/uKH/dzYu4y/+fdvprM1W4aKRUROpNM7ZqCrLcu316/mk/c/z5ce7ueB53fy3951AW9b1TntJQiGRvN889HX+NyGTTjwJzdcyAfevEKXLhCRitHB2Fn6wcZd/PfvbeS1fUd549IF/NveZbz1vC7O6WwZC+98wXlu+wE2bNzJPX3bGDg0zFtXdfI/3nMRyxc1V/kZiEhSzPRgrIL+FAyN5vn7p7Zzxz9v5cWdhwBoakjR1Zal4M6ug0Ph6o3GFed28uFfeQOrz1mkXryIzKmZBr2Gbk5BY0OKmy5fwU2Xr+C1vUf5fy/vYfOuw+w7MoyZ0b2gkfPPaOWq85awsMxfKCAiMh0F/WlasbiZFYtXVLsMEZFJ6awbEZGEU9CLiCScgl5EJOEU9CIiCaegFxFJOAW9iEjCKehFRBJOQS8iknDz4hIIZjZA/E1Vp6IT2DOH5cwV1TU787UumL+1qa7ZSWJdZ7n7tF+6PS+C/nSYWd9MrvVQaaprduZrXTB/a1Nds1PPdWnoRkQk4RT0IiIJl4Sgv63aBUxCdc3OfK0L5m9tqmt26raumh+jFxGRqSWhRy8iIlOo6aA3s2vNbJOZ9ZvZrRVe93Ize9jMNprZ82b2kdD+STPbbmZPh5/rS+7z+6HWTWb262WsbauZPRvW3xfaFpnZg2a2OfxeGNrNzL4Y6nrGzC4rU03nl2yTp83soJl9tBrby8y+Zma7zey5krZZbx8zWxuW32xma8tU12fM7MWw7nvNrCO0rzSzYyXb7Ssl93lTeP37Q+2n9dVmk9Q169dtrv9eJ6nr7pKatprZ06G9kttrsmyo3j7m7jX5A6SAl4FzgAzwM+CCCq5/KXBZmG4DXgIuAD4J/O4Ey18QaswCZ4faU2WqbSvQOa7tz4Bbw/StwKfD9PXAPwIGrAYerdBrtxM4qxrbC3gbcBnw3KluH2ARsCX8XhimF5ahrmuAdJj+dEldK0uXG/c4j4VaLdR+XRnqmtXrVo6/14nqGjf/c8AfVmF7TZYNVdvHarlHfznQ7+5b3H0E+DawplIrd/cd7v5kmD4EvAD0THGXNcC33X3Y3V8B+omfQ6WsAe4M03cCN5S03+WxR4AOM1ta5lquBl5296k+JFe27eXuPwH2TbC+2WyfXwcedPd97j4IPAhcO9d1ufsGd8+Fm48Ay6Z6jFDbAnd/xOO0uKvkucxZXVOY7HWb87/XqeoKvfIbgW9N9Rhl2l6TZUPV9rFaDvoe4PWS29uYOmjLxsxWApcCj4am3w5vwb5WfHtGZet1YIOZPWFm60Nbt7vvCNM7ge4q1FV0Eyf+AVZ7e8Hst081tttvEff8is42s6fM7Mdm9tbQ1hNqqURds3ndKr293grscvfNJW0V317jsqFq+1gtB/28YGatwHeBj7r7QeDLwBuAS4AdxG8fK+1Kd78MuA64xczeVjoz9FyqcrqVmWWAdwN/G5rmw/Y6QTW3z2TM7BNADvhGaNoBrHD3S4GPAd80swUVLGnevW7j/CYndiYqvr0myIYxld7HajnotwPLS24vC20VY2YNxC/kN9z97wDcfZe75929APwVx4cbKlavu28Pv3cD94YadhWHZMLv3ZWuK7gOeNLdd4Uaq769gtlun4rVZ2YfBN4FvD8EBGFoZG+YfoJ4/Pu8UEPp8E5Z6jqF162S2ysN/Gvg7pJ6K7q9JsoGqriP1XLQPw6sMrOzQy/xJuD+Sq08jAHeDrzg7n9e0l46vv0eoHhGwP3ATWaWNbOzgVXEB4Hmuq4WM2srThMfzHsurL941H4tcF9JXTeHI/+rgQMlby/L4YSeVrW3V4nZbp8HgGvMbGEYtrgmtM0pM7sW+Djwbnc/WtLeZWapMH0O8fbZEmo7aGarwz56c8lzmcu6Zvu6VfLv9VeBF919bEimkttrsmygmvvY6RxdrvYP8dHql4j/O3+iwuu+kvit1zPA0+HneuDrwLOh/X5gacl9PhFq3cRpHtmfoq5ziM9o+BnwfHG7AIuBh4DNwA+ARaHdgP8d6noW6C3jNmsB9gLtJW0V317E/2h2AKPE457rTmX7EI+Z94efD5Wprn7icdriPvaVsOy/Ca/v08CTwG+UPE4vcfC+DHyJ8MHIOa5r1q/bXP+9TlRXaL8D+I/jlq3k9posG6q2j+mTsSIiCVfLQzciIjIDCnoRkYRT0IuIJJyCXkQk4RT0IiIJp6AXEUk4Bb2ISMIp6EVEEu7/A3uJ9fJ+qhDGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pylab.plot(val_loss_list)\n",
    "pylab.savefig('val_loss.png')\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "88.97796306610107"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1965"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmin(loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73.32035293579102"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(val_loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "234"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmin(val_loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.272430658340454, 4.370163917541504, 3.9003374576568604, 4.230057239532471, 0.061188384890556335, 2.7572925090789795, 2.117064952850342, 3.601868152618408, 4.145383358001709, 3.9251646995544434, 2.165618658065796, 2.304734945297241, 2.0696866512298584, 3.603445529937744, 3.789348602294922]\n"
     ]
    }
   ],
   "source": [
    "print(log_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.8893168886662768, 8.891377249859119, 7.0298736228671155, 8.289823015521035, 1.0310670041973704, 3.969524263073115, 2.8821382783566682, 6.0553009367647315, 7.946182931806407, 7.117683702249636, 2.9529637559010204, 3.165678702731222, 2.8146652151348976, 6.060078567728462, 6.650382030063029]\n"
     ]
    }
   ],
   "source": [
    "# Found standard deviations (ground truth is 10 and 1):\n",
    "print([math.exp(log_var) ** 0.5 for log_var in log_vars])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}