lightning_lr_find.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Microsoft Visual C++ Redistributable is not installed, this may lead to the DLL load failure.\n",
      "                 It can be downloaded at https://aka.ms/vs/16/release/vc_redist.x64.exe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\conda\\envs\\lightning\\lib\\site-packages\\pytorch_lightning\\utilities\\distributed.py:37: UserWarning: Unsupported `ReduceOp` for distributed computing.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "%load_ext tensorboard\n",
    "\n",
    "import shutil\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "from pytorch_lightning.metrics import functional as FM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "mpl.style.use('seaborn-pastel')\n",
    "# mpl.rcParams['axes.facecolor'] = 'white'\n",
    "# mpl.rcParams[\"figure.facecolor\"] = 'white'\n",
    "mpl.rcParams['axes.prop_cycle'] = mpl.cycler(color=['#f38181', '#fce38a', '#95e1d3']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://pytorch.org/docs/stable/notes/randomness.html\n",
    "torch.manual_seed(7)\n",
    "np.random.seed(7)\n",
    "torch.backends.cudnn.benchmark = False\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightningDataModule (Fashion MNIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MNISTDataset(pl.LightningDataModule):\n",
    "    def __init__(self, root_dir, batch_size):\n",
    "        super().__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.root_dir = root_dir\n",
    "        self.transforms = transforms.Compose([transforms.ToTensor()])\n",
    "        \n",
    "    def prepare_data(self):\n",
    "        self.train_ds = torchvision.datasets.FashionMNIST(\n",
    "            self.root_dir, train=True, transform=self.transforms, download=True)\n",
    "        \n",
    "    def setup(self, stage=None):\n",
    "        val_len = int(0.2 * len(self.train_ds))\n",
    "        train_len = len(self.train_ds) - val_len\n",
    "        self.train_ds, self.val_ds = torch.utils.data.random_split(\n",
    "            self.train_ds, [train_len, val_len],\n",
    "            generator=torch.Generator().manual_seed(7))\n",
    "    \n",
    "    def train_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(self.train_ds, batch_size=self.batch_size, shuffle=True)\n",
    "    \n",
    "    def val_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(self.val_ds, batch_size=self.batch_size, shuffle=False)\n",
    "    \n",
    "    def test_dataloader(self):\n",
    "        test_ds = torchvision.datasets.FashionMNIST(\n",
    "            self.root_dir, train=False, transform=self.transforms, download=True)\n",
    "        loader = torch.utils.data.DataLoader(dataset=test_ds, batch_size=self.batch_size, shuffle=False)\n",
    "        return loader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightningModule (LeNet classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LitClassifier(pl.LightningModule):\n",
    "    def __init__(self, num_classes=10):\n",
    "        super().__init__()\n",
    "        self.lr = 2e-3\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(1, 6, 5, padding=2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1   = nn.Linear(16*5*5, 120)\n",
    "        self.fc2   = nn.Linear(120, 84)\n",
    "        self.fc3   = nn.Linear(84, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))\n",
    "        x = F.max_pool2d(F.relu(self.conv2(x)), (2, 2))\n",
    "        x = x.view(-1, self.num_flat_features(x))\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "    \n",
    "    def num_flat_features(self, x):\n",
    "        # Get the number of features in a batch of tensors `x`.\n",
    "        size = x.size()[1:]\n",
    "        return np.prod(size)\n",
    "    \n",
    "    def training_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        y_hat = self(x)\n",
    "        \n",
    "        acc = FM.accuracy(y_hat, y)\n",
    "        loss = F.cross_entropy(y_hat, y)\n",
    "        \n",
    "        result = pl.TrainResult(loss)\n",
    "        result.log_dict({'train_acc': acc, 'train_loss': loss}, on_step=True, on_epoch=False)\n",
    "        return result\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        y_hat = self(x)\n",
    "        \n",
    "        val_acc = FM.accuracy(y_hat, y)\n",
    "        val_loss = F.cross_entropy(y_hat, y)\n",
    "        \n",
    "        result = pl.EvalResult()\n",
    "        result.log_dict({'val_acc': val_acc, 'val_loss': val_loss}, reduce_fx=torch.mean)\n",
    "#         result.log('val/epoch_end_metric', y_hat, on_step=False, on_epoch=True, prog_bar=True, reduce_fx=torch.mean)\n",
    "        return result\n",
    "\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        result = self.validation_step(batch, batch_idx)\n",
    "        result.rename_keys({'val_acc': 'test_acc', 'val_loss': 'test_loss'})\n",
    "        return result\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        return torch.optim.Adam(self.parameters(), lr=self.lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    shutil.rmtree('logs/')\n",
    "except FileNotFoundError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 6006 (pid 11324), started 2:38:33 ago. (Use '!kill 11324' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-a8d857c494cf3fab\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-a8d857c494cf3fab\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          url.port = 6006;\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%tensorboard --logdir logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 5\n",
    "batch_size = 64\n",
    "root_dir = 'data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 1e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default model LR: 1e-05\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\conda\\envs\\lightning\\lib\\site-packages\\pytorch_lightning\\utilities\\distributed.py:37: UserWarning: Could not log computational graph since the `model.example_input_array` attribute is not set or `input_array` was not given\n",
      "  warnings.warn(*args, **kwargs)\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | conv1 | Conv2d | 156   \n",
      "1 | conv2 | Conv2d | 2 K   \n",
      "2 | fc1   | Linear | 48 K  \n",
      "3 | fc2   | Linear | 10 K  \n",
      "4 | fc3   | Linear | 850   \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "caf9ab9688474e4f857bb971503f86e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\conda\\envs\\lightning\\lib\\site-packages\\pytorch_lightning\\utilities\\distributed.py:37: RuntimeWarning: The metric you returned None must be a `torch.Tensor` instance, checkpoint not saved HINT: what is the value of loss in validation_epoch_end()?\n",
      "  warnings.warn(*args, **kwargs)\n",
      "D:\\conda\\envs\\lightning\\lib\\site-packages\\pytorch_lightning\\utilities\\distributed.py:37: RuntimeWarning: Can save best model only with loss available, skipping.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving latest checkpoint..\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = LitClassifier()\n",
    "dataset = MNISTDataset(root_dir, batch_size)\n",
    "logger  = TensorBoardLogger(save_dir='logs/1e-5')\n",
    "trainer = pl.Trainer(gpus=1, max_epochs=num_epochs, logger=logger, auto_lr_find=False, val_check_interval=0.5)\n",
    "\n",
    "model.lr = 1e-5\n",
    "\n",
    "print(f'Default model LR: {model.lr}')\n",
    "\n",
    "trainer.fit(model, dataset);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | conv1 | Conv2d | 156   \n",
      "1 | conv2 | Conv2d | 2 K   \n",
      "2 | fc1   | Linear | 48 K  \n",
      "3 | fc2   | Linear | 10 K  \n",
      "4 | fc3   | Linear | 850   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default model LR: 0.0001\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9977a0dad2f3449eab188cb3dfa011b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving latest checkpoint..\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = LitClassifier()\n",
    "dataset = MNISTDataset(root_dir, batch_size)\n",
    "logger  = TensorBoardLogger(save_dir='logs/1e-4')\n",
    "trainer = pl.Trainer(gpus=1, max_epochs=num_epochs, logger=logger, auto_lr_find=False, val_check_interval=0.5)\n",
    "\n",
    "model.lr = 1e-4\n",
    "\n",
    "print(f'Default model LR: {model.lr}')\n",
    "\n",
    "trainer.fit(model, dataset);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 1e-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | conv1 | Conv2d | 156   \n",
      "1 | conv2 | Conv2d | 2 K   \n",
      "2 | fc1   | Linear | 48 K  \n",
      "3 | fc2   | Linear | 10 K  \n",
      "4 | fc3   | Linear | 850   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default model LR: 0.1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1915b628a2c4b26965b010cb1613435",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving latest checkpoint..\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = LitClassifier()\n",
    "dataset = MNISTDataset(root_dir, batch_size)\n",
    "logger  = TensorBoardLogger(save_dir='logs/1e-1')\n",
    "trainer = pl.Trainer(gpus=1, max_epochs=num_epochs, logger=logger, auto_lr_find=False, val_check_interval=50)\n",
    "\n",
    "model.lr = 1e-1\n",
    "\n",
    "print(f'Default model LR: {model.lr}')\n",
    "\n",
    "trainer.fit(model, dataset);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainer.test(model, ckpt_path='best', verbose=True, datamodule=dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## find_lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | conv1 | Conv2d | 156   \n",
      "1 | conv2 | Conv2d | 2 K   \n",
      "2 | fc1   | Linear | 48 K  \n",
      "3 | fc2   | Linear | 10 K  \n",
      "4 | fc3   | Linear | 850   \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d37a99f1bf14418995a0d1622858b3b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Finding best initial lr', style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving latest checkpoint..\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | conv1 | Conv2d | 156   \n",
      "1 | conv2 | Conv2d | 2 K   \n",
      "2 | fc1   | Linear | 48 K  \n",
      "3 | fc2   | Linear | 10 K  \n",
      "4 | fc3   | Linear | 850   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Auto-find model LR: 0.036307805477010104\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e1c882471964f5fbfd4935bfba8521b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnTUlEQVR4nO3de5RV9X338fd373Pm4oxchBFxGBgYBgEjiqJizM2kyROTNiZNzOVJbHMrT56ap7G1XU2zsvr0WV3Pk/7R2DY1rXXFJE1raoyaFWu8xJWQi9eCBAUc5C4gICC3wYGZc/k+f+wzwzCegRk4++xz+bzWOmvO2fu39/7OZtifs68/c3dERERGCpIuQEREKpMCQkREilJAiIhIUQoIEREpSgEhIiJFKSBERKSoVNIFlNLUqVO9s7Mz6TJERKrGc889t9/d24qNq6mA6OzsZOXKlUmXISJSNczs5dHG6RCTiIgUpYAQEZGiFBAiIlKUAkJERIpSQIiISFEKCBERKaqmLnOV2uG5HOTzQy8f/r6/H/r78YGBaJh79Bpkhk2cSDB5MtbQgLvDwADe34+FITQ0QCqFmUXLcodcLmozMIA1NEBTExac+P40+Fj8wWlE6kFsAWFmHcD3gAuAPHCnu//DiDY3AH9dGJ8FbnH3Jwrj3gv8AxAC33L3v4mrVnev2P/47o4fPYofOwbHjkUbRwCz6DW4gQRs8mSC887DUm/8Z/V8Ppp++AY1DKMNZiqFpdNnX2smQ37/fvK7d5PbvRuOH4eGhmiDW6QmzCCbxY8dw/v68NdfH/pJJnPW9QDQ3AwDA1EAFFv+UPFF+kVpaIh+ZrPReh6cJghOrH8zrKkJO+ccrKUFa2yMfud0Omo3fP7u0fofFka4QzodtU+lIAiwIDgRapkMuEfzbmnBmpujtmEY/WxuPrHslpaK/TuW6hTnHkQWuNXdV5nZucBzZva4u784rM3PgAfd3c1sEXAvMN/MQuCbwLuBncAKM3twxLQl4e4cu+suwvZ20ldfTXDeeWc+r/5+/MiRExthOLERCYITG4LCxmDkN1g/fhzv7cWPHiV/4AC57dvJb98ebTDHygybNCkKicH5D250T9U5VFMTwXnnEUyaBGGIZ7PRhjGbjd7ncif/Tu4nfaP3vr6o/aDGRqyl5cSGcPi4oRVWCKnBDVxzM8HUqdHnpiYIw2gjW3iZWdR+cCPc0PCGjTUAuRz5Q4fwgwfJ9/ZiDQ1D8/Rs9qQN79Bqa2gY2rD7wAAcP44fPx7NMwyHQndoAz8Ysu7Rv1thHecPHYJM5sTezbB/l8FwsTCM1s9gAPX2ks9khoLI8/no3zGdHmqT274d+vpO/W/f2EgwZcqJ19SpWOF9sS8NIqcT21+Nu+8Gdhfe95pZD9AOvDiszdFhk7QAg/9jrwI2ufsWADO7B7hh+LQlMzBAcP75ZFatIrNiBeH8+TQsXUrQ0XHab2N+7Bj5gwfJbd9ObuNGci+/fPJG4VTMoqAIgugbbpHpbOJEwjlzCNrbT3x7bGwsLLywgRrcQObz5A8eJL9/P37wYLQhLLQJLrwQa22N5jEsOMjno0M5mQx+5EgUSnv2QD4/9C2VVCqaprHxRDAM1l/YcAeDG+DmZmzyZMLp07HJkxP9NhvOmJHYsuPi+XwUXIPBnclEe5Z9feR7e/EDB8i/9hq57dvJrllz0rQ2eXIUGpMmYRMnYpMmkersjEJcZBRl+VphZp3AYuDZIuM+BHwNOB94f2FwO7BjWLOdwNWjzHsZsAxg5syZ46+tsZGmD3yA/HXXkVmxgsxzz3Fs/XqC9nbSS5eSWrDgpGPRub17GVi+PAqD48eHhgdtbdEeyPTpUfvBaQa/ZQ5uiId/gx0MhsK3YWtqws49N9qYT5xIcO654/pdanGjKCdYEMA55zCW2PWBAfIHDkSH/Pbvx/fvJ//aa2R27hz6u+03I+joIDV/Pqn586O9R5FhLO4+qc2sFfgl8H/d/YFTtHsb8Jfu/ltmdiPw39z984VxNwFXufv/OtWylixZ4mf7LCYfGCDz/PNknnkGP3gQmppIzZ1L2NVFbts2ss8/D42NpC6+OPpGNnkywQUX6D+XVA0fGCC/bx/ZjRvJvfQS+VdfBSCYPp3U/PmkL79cexZ1xMyec/clxcbFugdhZmngfuDuU4UDgLv/ysy6zGwq0R5Dx7DRM4Bd8VV6gjU00HDllaSvuILcxo1k168nt2kT2bVrIQxJL11Kw1vegp1zTjnKESk5a2ggbG8nbG+Hd7yD/IEDZNevJ7t+PQPLlzPw5JM0XHMN6aVLTxzSlLoU2x6ERQeg/xU44O63jNJmLrC5cJL6cuA/icIgBDYA7wJeAVYA/93d151qmaXYgyjG3cnv2YO1tBBMmFDy+YtUivz+/fT//Ofk1q/HzjmHxve/n9SCBUmXJTFKag/iWuAmYI2ZrS4M+wowE8Dd7wA+DPyemWWAY8DHPEqsrJl9EXiMKCy+fbpwiJOZEU6fntTiRcommDqV5o9+lNwrr9D/yCMc/+EPSV99NQ2/9VvR1VdSV2I/B1FOce1BiNQjz2YZePxxMitWEHR00HTjjQStrUmXJSV2qj0IPWpDRIqyVIrG66+n8cMfJr9nD8fvvju6N0TqhgJCRE4pffHFNH30o+T37ePYD34QXaotdUEBISKnlerqovGDHyT/8sscf+CB6KY9qXkKCBEZk/Sb3kTDe95Dbv16Mk8/nXQ5UgYKCBEZs4alSwm7uqIbSUv1QEWpWAoIERmX9JvfjL/+evRUAalpCggRGZews5PgwgsZePppnYuocQoIERkXM6Ph2mvxgwfJ9vQkXY7ESAEhIuMWXnQRNmUKmSefpJZutpWTKSBEZNwsCGi45hrye/aQ27Il6XIkJgoIETkjqUWLsJYWMnq8Tc1SQIjIGbFUitSiReQ2biR/9OjpJ5Cqo4AQkTOWvuwyyOff0MWp1AYFhIicsaCtjaC9nezq1TpZXYMUECJyVtKLF5Pft4/8rrJ0+ihlpIAQkbOSuvhiSKXIrF6ddClSYgoIETkr1thIauFCsmvX6vlMNUYBISJnLX3ZZdDfrzura4wCQkTOWjBrFjZxItkXX0y6FCkhBYSInDUzI9XdTW7rVvU4V0MUECJSEmF3N2Qy5LZtS7oUKZHYAsLMOsxsuZn1mNk6M/tSkTafNLMXCq+nzOzSYeO2mdkaM1ttZrqXX6TChZ2dkEqR27Qp6VKkROLcg8gCt7r7AmApcLOZLRzRZivwdndfBPw1cOeI8de5+2XuviTGOkWkBCydJpw9m+zGjbpprkbEFhDuvtvdVxXe9wI9QPuINk+5+8HCx2eAGXHVIyLxS3V34wcP4q+9lnQpUgJlOQdhZp3AYuDZUzT7HPDIsM8O/NTMnjOzZTGWJyIlEnZ3A5DduDHhSqQUYg8IM2sF7gducfcjo7S5jigg/nzY4Gvd/XLgeqLDU28bZdplZrbSzFbu27evxNWLyHgEEycSnH8+OQVETYg1IMwsTRQOd7v7A6O0WQR8C7jB3Yf2S919V+HnXuBHwFXFpnf3O919ibsvaWtrK/WvICLjFM6dS277dry/P+lS5CzFeRWTAXcBPe5+2yhtZgIPADe5+4Zhw1vM7NzB98B7gLVx1SoipZPq7oZ8Xj3N1YBUjPO+FrgJWGNmqwvDvgLMBHD3O4C/BKYA/xTlCdnCFUvTgB8VhqWA77v7ozHWKiIlEnR0QHMzmTVrSC1YkHQ5chZiCwh3fwKw07T5PPD5IsO3AJe+cQoRqXQWBKSvvJLMr35Fbs8ewgsuSLokOUO6k1pESq5h6VJoamLgF79IuhQ5CwoIESk5a2qi4ZpryG3YQO6VV5IuR86QAkJEYpG+6ipobtZeRBVTQIhILKyxkYY3v5nc5s3kduxIuhw5AwoIEYlN+sorsZYWBp56KulS5AwoIEQkNtbQQOqSS8ht3Ij39SVdjoyTAkJEYpVatAjyefU2V4UUECISq2DaNIK2NjJr1iRdioyTAkJEYmVmpC65hPyOHeQPHUq6HBkHBYSIxC71pjcBkNVeRFVRQIhI7IJJkwhmziSzZo16m6siCggRKYv0JZfg+/eT37Mn6VJkjBQQIlIWqYULIQh0mKmKKCBEpCysuZlw7lyyPT06zFQlFBAiUjapBQvww4fJ79qVdCkyBgoIESmb1Lx50WGmnp6kS5ExUECISNlYczNhZ6cOM1UJBYSIlFVq4UL84EHyr76adClyGgoIESmr1EUXgZkOM1UBBYSIlJW1tBDOnKmAqAIKCBEpu3DBguimuX37ki5FTiG2gDCzDjNbbmY9ZrbOzL5UpM0nzeyFwuspM7t02Lj3mtlLZrbJzL4cV50iUn6p+fMBtBdR4eLcg8gCt7r7AmApcLOZLRzRZivwdndfBPw1cCeAmYXAN4HrgYXAJ4pMKyJVKpgwgaC9neyGDUmXIqcQW0C4+253X1V43wv0AO0j2jzl7gcLH58BZhTeXwVscvct7j4A3APcEFetIlJ+qblzye/apZ7mKlhZzkGYWSewGHj2FM0+BzxSeN8ODO/lfCcjwkVEqls4Zw4A2a1bE65ERhN7QJhZK3A/cIu7HxmlzXVEAfHng4OKNCt6V42ZLTOzlWa2cp9OeIlUjaC9HRobyW3ZknQpMopYA8LM0kThcLe7PzBKm0XAt4Ab3P21wuCdQMewZjOAog9vcfc73X2Juy9pa2srXfEiEisLAsLOTnJbtuiu6goV51VMBtwF9Lj7baO0mQk8ANzk7sPPVq0Aus1stpk1AB8HHoyrVhFJRqqrCz98GD9wIOlSpIhUjPO+FrgJWGNmqwvDvgLMBHD3O4C/BKYA/xTlCdnC3kDWzL4IPAaEwLfdfV2MtYpIAobOQ2zZQsOUKQlXIyPFFhDu/gTFzyUMb/N54POjjHsYeDiG0kSkQtjkydikSdF5iCuvTLocGUF3UotIYsyMcM4cctu24blc0uXICAoIEUlUas4c6O8n/8orSZdSlfJHj8Z2qbACQkQSFc6eDUTnIWR88ocOcey73+X4fffhAwMln78CQkQSZc3NBBdeqPshxim3dy/HvvMdvK+P5o9/HGtoKPkyFBAikrhw9uzosRv9/UmXUhVyO3dy7LvfBXeaP/1pwo6O005zJhQQIpK4cPZsyOfJvfxy0qVUvOxLL3Hse9/Dmptp/sxnCM8/P7ZlKSBEJHFhRweEITk9l+mUMitXcvzeewnOP5/mz36WYPLkWJcX541yIiJjYuk0YUcHuW3bki6lYg38+tcMLF9O2N1N04c/HMs5h5G0ByEiFSGcPZv8q6+Sf/31pEupOJm1axlYvpzUokU0fexjZQkHUECISIUYvNxVexEny73yCv0PPkgwcyaNv/M7WFC+zbYCQkQqQnDhhdHjv3UeYkj+yBGO/+AHWGsrzTfeiIVhWZevgBCRimBBQDhrlgKiwLNZjt97Lz4wEB1Wamkpew0KCBGpGOHs2fjBg+QPHUq6lMQNPP44+V27aLrhBsJp0xKpQQEhIhVj6DxEne9FZNasIbNiBemlS0ktWJBYHQoIEakYQVsb1tpa1wGR27uX/oceIujooOFd70q0FgWEiFQMMyPs7ia7fj35w4eTLqfs3J3+Bx/EGhpo+shHyn5SeiQFhIhUlIa3vhWAgZ//POFKyi+3dSv5XbtouO46gnPPTbocBYSIVJZg0iTSS5eSXbOGXJ31EZF58kmstZXUokVJlwIoIESkAjVcey3W2kr/Y4/h7kmXUxa53bvJbd1K+uqrsVRlPAVJASEiFccaG2m47jryO3eSffHFpMspi8xTT0FjI+krrki6lCEKCBGpSKlLLyWYNo2BX/4y6VJilz9wgOyLL5K+4gqsqSnpcoYoIESkIlkQkFq8GN+/n/yBA0mXE6uBJ5+EICB99dVJl3KSMQWEmbWYWVB4P8/MPmBm6dNM02Fmy82sx8zWmdmXirSZb2ZPm1m/mf3piHHbzGyNma02s5Xj+aVEpDak5s4FILtpU8KVxGfgmWfI/uY3pJcsqYgrl4Yb6x7Er4AmM2sHfgZ8BvjuaabJAre6+wJgKXCzmS0c0eYA8EfA344yj+vc/TJ3XzLGOkWkhgTnnYeddx65Gg2IzKpVDPz0p4Tz59Pw7ncnXc4bjDUgzN37gN8F/tHdPwSM3NifxN13u/uqwvteoAdoH9Fmr7uvADLjrlxE6kJq7lxy27bh2WzSpZRUZu1a+h96iHDu3KgDoDI+xnusxhwQZnYN8EngJ4VhY74Oy8w6gcXAs+OozYGfmtlzZrbsFPNeZmYrzWzlvn37xjF7EakGYVcXZLM111/1wC9+QTB9Ok0JPMZ7rMYaELcAfwH8yN3XmdkcYPlYJjSzVuB+4BZ3PzKO2q5198uB64kOT72tWCN3v9Pdl7j7kra2tnHMXkSqQdjZGfVXXWOHmfzYMcL2dix9ytO5iRrTXoC7/xL4JUDhZPV+d/+j001XOJF9P3C3uz8wnsLcfVfh514z+xFwFdG5EBGpI5ZOE86aRXbzZhqTLqZE3B36+6Gxsn+jsV7F9H0zm2BmLcCLwEtm9menmcaAu4Aed79tPEUVrpo6d/A98B5g7XjmISK1I5w7N7rctVb6ichmIZ/HaiEggIWFw0MfBB4GZgI3nWaaawtt3lm4VHW1mb3PzL5gZl8AMLMLzGwn8CfAV81sp5lNAKYBT5jZ88B/AT9x90fH/duJSE0YvNy1Vg4zeX8/QMUHxFhPNKcLh4s+CNzu7hkzO+UDUtz9CcBO02YPMKPIqCPApWOsTURqnE2Zgk2aRHbzZtJLauCq90JA1MQhJuBfgG1AC/ArM5tFtBEXEYmdmZHq7ia3eTPe15d0OWetWvYgxhQQ7v4Nd2939/d55GXguphrExEZkrriCshmyTz3XNKlnLWaCggzm2hmtw3eb2BmXyfamxARKYvw/PMJ58whs2IFnsslXc7ZqbFDTN8GeoGPFl5HgO/EVZSISDHppUvxo0fJrluXdClnpVr2IMZ6krrL3T887PP/MbPVMdQjIjKqsKsLmzqVzLPPkrrkEqKr6atPtQTEWPcgjpnZWwY/mNm1wLF4ShIRKc7MaLj6avK7d5Pfvj3pcs5cjR1i+gLwzcIjuLcBtwP/I7aqRERGkVq0CJqbGXjmmaRLOWPe3w9hWDFdi45mrFcxPe/ulwKLgEXuvhh4Z6yViYgUYek06cWLyW3YULWXvHp/f8UfXoJx9ijn7keGPXDvT2KoR0TktFIXXwzuZF96KelSzkwVPIcJzq7L0eo8OyQiVS+44ILozuqenqRLOSM1uQcxwikftSEiEhczIzV/PrktW/Djx5MuZ9y8FvYgzKzXzI4UefUCF5apRhGRN0gtWAD5PNmNG5MuZfyqZA/ilKfQ3b2yetAWESkIZszAWlvJ9vSQvuSSpMsZF+/vJ6iCgKi8TlBFRMZg6DDTpk14prq6ta+HcxAiIokK58+P+quuon4iqqU3OVBAiEgVCzs7obm5uq5mqpLe5EABISJVzIKA1EUXkd24sWqe8OpV8pgNUECISJVLXXQR9PeT27Yt6VLGpkoe1AcKCBGpcuGcOZBKkauSu6qr5UmuoIAQkSpn6TRhVxfZDRuiE8AVToeYRETKKHXRRfiRI+T37Em6lNPTHgSYWYeZLTezHjNbZ2ZfKtJmvpk9bWb9ZvanI8a918xeMrNNZvbluOoUkeqX6u4Gs6p4eJ8OMUWywK3uvgBYCtxsZgtHtDkA/BHwt8MHmlkIfBO4HlgIfKLItCIiAFhLC8GMGVVxHkIBAbj7bndfVXjfC/QA7SPa7HX3FcDI2yCvAja5+xZ3HwDuAW6Iq1YRqX6piy4i/+qr5A8dSrqUU9M5iJOZWSewGHh2jJO0AzuGfd7JiHAZNu9lZrbSzFbu27fvrOoUkeqVuugiALIbNiRcyalVS29yUIaAMLNW4H7glmGdDZ12siLDil6e4O53uvsSd1/S1tZ2pmWKSJULpkzBpk6t+MNM1fKob4g5IMwsTRQOd7v7A+OYdCfQMezzDGBXKWsTkdqTmjeP3MsvV3YfEVXyoD6I9yomA+4Cetz9tnFOvgLoNrPZZtYAfBx4sNQ1ikhtSc2dC/l8Rd9VXS1PcoXT9Adxlq4FbgLWmNnqwrCvADMB3P0OM7sAWAlMAPJmdguw0N2PmNkXgceAEPi2u6+LsVYRqQFBRwc0NJDdvJnU/PlJl1NUNR1iii0g3P0JTtNvtbvvITp8VGzcw8DDMZQmIjXKwpCws5Pc5s24O9GBjAozMIBNmJB0FWOiO6lFpKakurrwQ4fwAweSLqUoP368ag4xKSBEpKaEXV0AZDdvTriS4qrpEJMCQkRqSnDeedjkyeQqMCAGe5PTHoSISEJSXV3ktm2rvE6Ecrmq6U0OFBAiUoPCri7IZMht3550KSeppkd9gwJCRGpQ2NkJQVB5h5mq6EF9oIAQkRpkjY0EHR3ktmxJupSTDN7hrYAQEUlQqquL/J495I8eTbqUIUOHmBoaki1kjBQQIlKTwjlzAMht3ZpwJcMMDABgTU0JFzI2CggRqUnB9OnQ3FxRh5mqqbMgUECISI0yM1KzZw89dqMS6ComEZEKEc6Zgx89Sr5SOhPTSWoRkcowdB6iQg4zeX8/BAGEYdKljIkCQkRqVjBpEjZlSuUExMAANDVV5lNmi1BAiEjt2ryZxkceoemzn8WDACZMgD/8Q0jqBroqeg4TKCBEpFY98ggsWkT42GNYfz/mDr298K1vwaJF0fgyq6be5EABISK1aPNm+MhHoK8Py2ZPHpfJQF9fNL7MexJ+/HjV3CQHCggRqUVf/3oUBKeSycDf/V156hk0MKA9CBGRRP37v48tIP7t38pTT4H391fNXdSggBCRWjTW5y+V8TlN2c2b8YMHCaZMKdsyz5YCQkRqT2tradudJe/ro//HPyZoayN9zTVlWWYpxBYQZtZhZsvNrMfM1pnZl4q0MTP7hpltMrMXzOzyYeO2mdkaM1ttZivjqlNEatCnPgXp9KnbpNNw002xl+LuHH/oIbyvj8YPfQg7XV0VJM49iCxwq7svAJYCN5vZwhFtrge6C69lwD+PGH+du1/m7ktirFNEas2tt44tIP74j2MvJbt6Nbn162l45zsJL7gg9uWVUmwB4e673X1V4X0v0AO0j2h2A/A9jzwDTDKz6XHVJCJ1oqsL7rsPzjnnDUHhqVQ0/L77onYx8nye/scfJ5w1q6oOLQ0qyzkIM+sEFgPPjhjVDuwY9nknJ0LEgZ+a2XNmtuwU815mZivNbOW+Snkgl4gk7/rr4YUXYNkymDABDwK8sZHc+98fDb/++thLyL/6Khw/Turyy6vm8RrDxR4QZtYK3A/c4u5HRo4uMsngc3mvdffLiQ5D3Wxmbys2f3e/092XuPuStra2ktUtIjWgqwtuvx0OH8ZyOY7dfjsDH/xg7HsOg3I7ou+/4cyZZVleqcUaEGaWJgqHu939gSJNdgIdwz7PAHYBuPvgz73Aj4Cr4qxVRGpfOGsW+VdewUfeXR2T/I4d2IQJBBMnlmV5pRbnVUwG3AX0uPttozR7EPi9wtVMS4HD7r7bzFrM7NzCfFqA9wBr46pVROpDOHMm5HLkX3kl9mW5O7nt26t27wEgFeO8rwVuAtaY2erCsK8AMwHc/Q7gYeB9wCagD/hMod004EeFY3Yp4Pvu/miMtYpIHRjcWOdefplw1qxYl+WHD+O9vQQdHadvXKFiCwh3f4Li5xiGt3Hg5iLDtwCXxlSaiNQpa24mmDaN3PbtsS9r6PxDFQeE7qQWkboSzpxJbscOPJeLdTm57duhsZHg/PNjXU6cFBAiUlfCWbMgk4n9PER+xw7CGTOwoHo3s9VbuYjIGQjnzIEgILtxY0nn6319REfNwY8dI793b1UfXgIFhIjUGWtqig4zbdhQsnnmdu/m9b//e47fcw+ezQ6dfwiq+AomUECISB0K580jv28f+YMHz3pePjDA8QcegFSK3MaNHL/3XnJbt0IQELaPfLpQdVFAiEjdSXV3A5TkMFP/o4/ir71G04030vjbv01u0yYyzz5LMH16VT25tRgFhIjUnWDKFGzKlLM+zJRZt47s6tWk3/IWUrNnk778cho/8AEAws7OElSarDhvlBMRqVip7m4yK1ZE3YCeQT/R+d5e+h96iGDGDBre/vah4enLLouuXpo0qYTVJkN7ECJSl1Lz5kEuR27LljOafuDnP4dslqYbbsDC8KRxwdSpWKr6v38rIESkLgUdHdDUdEbnIXI7d5J9/nnSV19dVX1Mj5cCQkTqkoUhqa4uchs3Dt2/MBbuTv+jj2KtrTS89a0xVpg8BYSI1K1w3jz89dfJ79w55mmyL7xAftcuGt71rjM6d1FNFBAiUrdS3d0QhmRffHFM7T2bZeBnPyO48EJSixbFXF3yFBAiUresqYmwq4tsT8+YDjPl9+/Hjx4lvXRpVXYhOl4KCBGpa6mLL8aPHBnTYSY/fBiAYPLkuMuqCAoIEalrqXnzxnyYKX/oEEBN3OMwFgoIEalr1thIOHcu2RdfPO1hJj98GFIp7JxzylRdshQQIlL3UgsX4r295AtPYR1N/vBhbOLEujj/AAoIEZExH2byw4cJJk4sU1XJU0CISN2zxkbC7u7oMFM+P2o7P3Sobs4/gAJCRAQoHGY6enTUw0yeyeB9fdqDKAUz6zCz5WbWY2brzOxLRdqYmX3DzDaZ2Qtmdvmwce81s5cK474cV50iIlC4aS4IyI7yCPDBS1xNAVESWeBWd18ALAVuNrOFI9pcD3QXXsuAfwYwsxD4ZmH8QuATRaYVESkZa2wk7OwcNSAGL3HVHkQJuPtud19VeN8L9AAj+9+7AfieR54BJpnZdOAqYJO7b3H3AeCeQlsRkdiE8+bhr71Gfv/+N4wb2oPQOYjSMrNOYDHw7IhR7cDwA347C8NGGy4iEpvUvHkARfci8ocPgxl27rnlLisxsQeEmbUC9wO3uPuRkaOLTOKnGF5s/svMbKWZrdy3b9/ZFSsidS2YNIlg2rSiAeGHD2MTJmBB/VzbE+tvamZponC4290fKNJkJ9Ax7PMMYNcphr+Bu9/p7kvcfUlbW1tpCheRuhXOm0d+xw68r++k4flDh+rqBDXEexWTAXcBPe5+2yjNHgR+r3A101LgsLvvBlYA3WY228wagI8X2oqIxCo1bx64k9206aThfvgwQR2dfwCIs9PUa4GbgDVmtrow7CvATAB3vwN4GHgfsAnoAz5TGJc1sy8CjwEh8G13XxdjrSIiAAQXXoi1tpLbsIF0oc8Hz+Xw3t6624OILSDc/QmKn0sY3saBm0cZ9zBRgIiIlI2ZEc6bR3btWjybxVIpvLcX3OvqElfQndQiIm+QmjcPBgbIbdkCRI/YgPq6SQ4UECIibxB2dWEtLWRWrQIKl7hC3Z2DUECIiIxgYUjqssvIbdxI/vDhunzMBiggRESKSl9xBbiTWbUq6geipQVLxXldT+VRQIiIFBFMmhT1NPeb35A/cKDu9h5AASEiMqr0kiXRI8Bffrnuzj+AAkJEZFTh3LlDew7agxARkSEWBKQvj7qpUUCIiMhJ0pdfTjhrFqnZs5Mupezq65S8iMg4WUsLzb//+0mXkQjtQYiISFEKCBERKUoBISIiRSkgRESkKAWEiIgUpYAQEZGiFBAiIlKUAkJERIqyqNfP2mBm+4CXk66jxKYC+5MuoopofY2P1tf41OL6muXubcVG1FRA1CIzW+nuS5Kuo1pofY2P1tf41Nv60iEmEREpSgEhIiJFKSAq351JF1BltL7GR+trfOpqfekchIiIFKU9CBERKUoBISIiRSkgRESkKAVEFTOzt5rZHWb2LTN7Kul6Kp2ZvcPMfl1YZ+9Iup5KZ2YLCuvqPjP7n0nXU+nMbI6Z3WVm9yVdS6koIBJiZt82s71mtnbE8Pea2UtmtsnMvnyqebj7r939C8BDwL/GWW/SSrG+AAeOAk3AzrhqrQQl+vvqKfx9fRSo6ZvDSrS+trj75+KttLx0FVNCzOxtRBur77n7mwrDQmAD8G6iDdgK4BNACHxtxCw+6+57C9PdC3ze3Y+UqfyyK8X6Ava7e97MpgG3ufsny1V/uZXq78vMPgB8Gbjd3b9frvrLrcT/H+9z94+Uq/Y4pZIuoF65+6/MrHPE4KuATe6+BcDM7gFucPevAb9dbD5mNhM4XMvhAKVbXwUHgcZYCq0QpVpf7v4g8KCZ/QSo2YAo8d9XzdAhpsrSDuwY9nlnYdipfA74TmwVVbZxrS8z+10z+xfg34DbY66tEo13fb3DzL5RWGcPx11cBRrv+ppiZncAi83sL+Iurhy0B1FZrMiwUx4DdPf/HVMt1WBc68vdHwAeiK+cijfe9fUL4BdxFVMFxru+XgO+EF855ac9iMqyE+gY9nkGsCuhWqqB1tf4aH2NT92vLwVEZVkBdJvZbDNrAD4OPJhwTZVM62t8tL7Gp+7XlwIiIWb2H8DTwEVmttPMPufuWeCLwGNAD3Cvu69Lss5KofU1Plpf46P1VZwucxURkaK0ByEiIkUpIEREpCgFhIiIFKWAEBGRohQQIiJSlAJCRESKUkBIzTOzo2VeXln75jCzSWb2h+VcptQHBYTIOJnZKZ9h5u5vLvMyJwEKCCk5PaxP6pKZdQHfBNqAPuAP3H29mf0O8FWgAXgN+KS7v2pmfwVcCHQC+81sAzATmFP4+ffu/o3CvI+6e2uh17q/AvYDbwKeAz7l7m5m7wNuK4xbBcxx95MeIW1mnwbeT9TBUUuhb4YfA5OBNPBVd/8x8DdAl5mtBh539z8zsz8j6uinEfhRnT/UUc6Uu+ulV02/gKNFhv0M6C68vxr4eeH9ZE48YeDzwNcL7/+KaAPfPOzzU0Qb4KlEYZIevjzgHcBhooe8BUSPcngL0QZ/BzC70O4/gIeK1PhpogfGnVf4nAImFN5PBTYRPXG0E1g7bLr3AHcWxgVEPQ6+Lel/B72q76U9CKk7ZtYKvBn4odnQE50HOxCaAfzAzKYT7UVsHTbpg+5+bNjnn7h7P9BvZnuBabyxK9P/cvedheWuJtqYHwW2uPvgvP8DWDZKuY+7+4HB0oH/V+j9LE/UN8G0ItO8p/D6TeFzK9AN/GqUZYgUpYCQehQAh9z9siLj/pGoO9IHhx0iGvT6iLb9w97nKP7/qVibYv0MjGb4Mj9JdEjsCnfPmNk2or2RkQz4mrv/yziWI/IGOkktdcej7lm3mtmNABa5tDB6IvBK4f3vx1TCemDOsC4uPzbG6SYCewvhcB0wqzC8Fzh3WLvHgM8W9pQws3YzO//sy5Z6oz0IqQfnmNnwQz+3EX0b/2cz+yrRCd97gOeJ9hh+aGavAM8As0tdjLsfK1yW+qiZ7Qf+a4yT3g38p5mtBFYTBQ3u/pqZPWlma4FHPDpJvQB4unAI7SjwKWBviX8VqXF63LdIAsys1d2PWrQF/yaw0d3/Lum6RIbTISaRZPxB4aT1OqJDRzpfIBVHexAiIlKU9iBERKQoBYSIiBSlgBARkaIUECIiUpQCQkREilJAiIhIUf8fxvIy5nnpeS0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# del model\n",
    "# del trainer\n",
    "# del dataset\n",
    "\n",
    "# model = LitClassifier()\n",
    "# trainer = get_trainer(logs_dir='logs/find_lr')\n",
    "# dataset = FashionMNISTDataset(root_dir='data/')\n",
    "\n",
    "model = LitClassifier(batch_size)\n",
    "dataset = MNISTDataset(root_dir, batch_size)\n",
    "logger  = TensorBoardLogger(save_dir='logs/find_lr')\n",
    "trainer = pl.Trainer(gpus=1, max_epochs=num_epochs, logger=logger, auto_lr_find=False, val_check_interval=0.5)\n",
    "\n",
    "lr_finder = trainer.lr_find(model, dataset)\n",
    "model.hparams.lr = lr_finder.suggestion()\n",
    "print(f'Auto-find model LR: {model.hparams.lr}')\n",
    "\n",
    "fig = lr_finder.plot(suggest=True)\n",
    "\n",
    "trainer.fit(model, dataset);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-16-6ebedab27258>:1: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
      "  fig.show()\n"
     ]
    }
   ],
   "source": [
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainer.test(model, ckpt_path='best', verbose=True, datamodule=dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainer\n",
    "\n",
    "# > <pytorch_lightning.trainer.trainer.Trainer at 0x1b886c29d90>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr_finder = trainer.tuner.lr_find(model, dataset)\n",
    "\n",
    "# > AttributeError: 'Trainer' object has no attribute 'tuner'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.lr = lr_finder.suggestion()\n",
    "# model.lr"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda lightning",
   "language": "python",
   "name": "lightning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}