generating_texts_with_rnns.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implements \"Generating Text with Recurrent Neural Networks\" (https://icml.cc/2011/papers/524_icmlpaper.pdf)\n",
    "\n",
    "<img src=\"assets/rnn.jpeg\" />\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "DATA\n",
      "Total characters: 40,888\n",
      "Unique characters: 100\n",
      "Training chunk length: 200\n",
      "Batch size: 64\n",
      "\n",
      "MODEL\n",
      "Input size: 100\n",
      "Hidden size: 256\n",
      "Factor size: 256\n",
      "Number of layers: 1\n",
      "Total parameters: 249,188\n",
      "\n",
      "TRAIN\n",
      "Learning rate: 0.001\n",
      "Number of epochs: 1000\n",
      "Gradient clipping: 0.25\n",
      "Weight decay: 0.01\n",
      "Device: cpu\n",
      "--------------------------------------------------\n",
      "\n",
      "epoch: 1/1000, loss: 4.6149, mfu: 15.89%\n",
      "epoch: 2/1000, loss: 4.6155, mfu: 17.87%\n",
      "epoch: 3/1000, loss: 4.6125, mfu: 16.56%\n",
      "epoch: 4/1000, loss: 4.6096, mfu: 16.88%\n",
      "epoch: 5/1000, loss: 4.6050, mfu: 17.78%\n",
      "epoch: 6/1000, loss: 4.5979, mfu: 16.04%\n",
      "epoch: 7/1000, loss: 4.5887, mfu: 16.60%\n",
      "epoch: 8/1000, loss: 4.5791, mfu: 18.02%\n",
      "epoch: 9/1000, loss: 4.5668, mfu: 17.15%\n",
      "epoch: 10/1000, loss: 4.5519, mfu: 17.59%\n",
      "epoch: 11/1000, loss: 4.5369, mfu: 17.27%\n",
      "epoch: 12/1000, loss: 4.5208, mfu: 17.73%\n",
      "epoch: 13/1000, loss: 4.5005, mfu: 16.14%\n",
      "epoch: 14/1000, loss: 4.4811, mfu: 17.17%\n",
      "epoch: 15/1000, loss: 4.4548, mfu: 18.22%\n",
      "epoch: 16/1000, loss: 4.4306, mfu: 16.46%\n",
      "epoch: 17/1000, loss: 4.3983, mfu: 17.02%\n",
      "epoch: 18/1000, loss: 4.3723, mfu: 17.65%\n",
      "epoch: 19/1000, loss: 4.3334, mfu: 17.67%\n",
      "epoch: 20/1000, loss: 4.3037, mfu: 17.69%\n",
      "epoch: 21/1000, loss: 4.2610, mfu: 17.06%\n",
      "epoch: 22/1000, loss: 4.2093, mfu: 16.88%\n",
      "epoch: 23/1000, loss: 4.1601, mfu: 16.53%\n",
      "epoch: 24/1000, loss: 4.1106, mfu: 17.30%\n",
      "epoch: 25/1000, loss: 4.0499, mfu: 17.55%\n",
      "epoch: 26/1000, loss: 3.9890, mfu: 18.19%\n",
      "epoch: 27/1000, loss: 3.9215, mfu: 18.02%\n",
      "epoch: 28/1000, loss: 3.8492, mfu: 18.82%\n",
      "epoch: 29/1000, loss: 3.7831, mfu: 17.24%\n",
      "epoch: 30/1000, loss: 3.7108, mfu: 17.53%\n",
      "epoch: 31/1000, loss: 3.6351, mfu: 16.68%\n",
      "epoch: 32/1000, loss: 3.5732, mfu: 18.00%\n",
      "epoch: 33/1000, loss: 3.5005, mfu: 18.09%\n",
      "epoch: 34/1000, loss: 3.4212, mfu: 16.89%\n",
      "epoch: 35/1000, loss: 3.3692, mfu: 16.72%\n",
      "epoch: 36/1000, loss: 3.3130, mfu: 16.86%\n",
      "epoch: 37/1000, loss: 3.2787, mfu: 16.02%\n",
      "epoch: 38/1000, loss: 3.2322, mfu: 17.15%\n",
      "epoch: 39/1000, loss: 3.1950, mfu: 18.70%\n",
      "epoch: 40/1000, loss: 3.1468, mfu: 16.66%\n",
      "epoch: 41/1000, loss: 3.0983, mfu: 17.43%\n",
      "epoch: 42/1000, loss: 3.0761, mfu: 17.50%\n",
      "epoch: 43/1000, loss: 3.0302, mfu: 15.78%\n",
      "epoch: 44/1000, loss: 3.0006, mfu: 16.42%\n",
      "epoch: 45/1000, loss: 2.9533, mfu: 18.75%\n",
      "epoch: 46/1000, loss: 2.9080, mfu: 17.22%\n",
      "epoch: 47/1000, loss: 2.8885, mfu: 15.42%\n",
      "epoch: 48/1000, loss: 2.8515, mfu: 16.45%\n",
      "epoch: 49/1000, loss: 2.8130, mfu: 16.69%\n",
      "epoch: 50/1000, loss: 2.8309, mfu: 15.62%\n",
      "epoch: 51/1000, loss: 2.7794, mfu: 17.50%\n",
      "epoch: 52/1000, loss: 2.7518, mfu: 17.64%\n",
      "epoch: 53/1000, loss: 2.7485, mfu: 17.46%\n",
      "epoch: 54/1000, loss: 2.7060, mfu: 16.25%\n",
      "epoch: 55/1000, loss: 2.6645, mfu: 18.40%\n",
      "epoch: 56/1000, loss: 2.6826, mfu: 17.03%\n",
      "epoch: 57/1000, loss: 2.6868, mfu: 16.20%\n",
      "epoch: 58/1000, loss: 2.6381, mfu: 15.45%\n",
      "epoch: 59/1000, loss: 2.6110, mfu: 16.99%\n",
      "epoch: 60/1000, loss: 2.6147, mfu: 18.21%\n",
      "epoch: 61/1000, loss: 2.5995, mfu: 17.90%\n",
      "epoch: 62/1000, loss: 2.6028, mfu: 17.69%\n",
      "epoch: 63/1000, loss: 2.5787, mfu: 17.53%\n",
      "epoch: 64/1000, loss: 2.5370, mfu: 17.47%\n",
      "epoch: 65/1000, loss: 2.5446, mfu: 17.65%\n",
      "epoch: 66/1000, loss: 2.5372, mfu: 18.44%\n",
      "epoch: 67/1000, loss: 2.5354, mfu: 16.21%\n",
      "epoch: 68/1000, loss: 2.5030, mfu: 15.05%\n",
      "epoch: 69/1000, loss: 2.4716, mfu: 17.40%\n",
      "epoch: 70/1000, loss: 2.4823, mfu: 17.20%\n",
      "epoch: 71/1000, loss: 2.4784, mfu: 17.20%\n",
      "epoch: 72/1000, loss: 2.4731, mfu: 18.24%\n",
      "epoch: 73/1000, loss: 2.4693, mfu: 17.97%\n",
      "epoch: 74/1000, loss: 2.4465, mfu: 17.87%\n",
      "epoch: 75/1000, loss: 2.4219, mfu: 18.09%\n",
      "epoch: 76/1000, loss: 2.4149, mfu: 16.56%\n",
      "epoch: 77/1000, loss: 2.4004, mfu: 17.62%\n",
      "epoch: 78/1000, loss: 2.3992, mfu: 16.87%\n",
      "epoch: 79/1000, loss: 2.3675, mfu: 17.54%\n",
      "epoch: 80/1000, loss: 2.3938, mfu: 16.03%\n",
      "epoch: 81/1000, loss: 2.3870, mfu: 16.23%\n",
      "epoch: 82/1000, loss: 2.3757, mfu: 14.54%\n",
      "epoch: 83/1000, loss: 2.3690, mfu: 16.60%\n",
      "epoch: 84/1000, loss: 2.3257, mfu: 17.86%\n",
      "epoch: 85/1000, loss: 2.3591, mfu: 15.82%\n",
      "epoch: 86/1000, loss: 2.3374, mfu: 17.57%\n",
      "epoch: 87/1000, loss: 2.3207, mfu: 16.83%\n",
      "epoch: 88/1000, loss: 2.3449, mfu: 16.52%\n",
      "epoch: 89/1000, loss: 2.3266, mfu: 17.43%\n",
      "epoch: 90/1000, loss: 2.3243, mfu: 16.98%\n",
      "epoch: 91/1000, loss: 2.3149, mfu: 17.30%\n",
      "epoch: 92/1000, loss: 2.3120, mfu: 15.35%\n",
      "epoch: 93/1000, loss: 2.3084, mfu: 17.96%\n",
      "epoch: 94/1000, loss: 2.3031, mfu: 17.48%\n",
      "epoch: 95/1000, loss: 2.3058, mfu: 17.10%\n",
      "epoch: 96/1000, loss: 2.2828, mfu: 18.32%\n",
      "epoch: 97/1000, loss: 2.2817, mfu: 18.29%\n",
      "epoch: 98/1000, loss: 2.2557, mfu: 14.31%\n",
      "epoch: 99/1000, loss: 2.2678, mfu: 16.90%\n",
      "epoch: 100/1000, loss: 2.2549, mfu: 18.01%\n",
      "epoch: 101/1000, loss: 2.2817, mfu: 17.25%\n",
      "epoch: 102/1000, loss: 2.2081, mfu: 16.71%\n",
      "epoch: 103/1000, loss: 2.2324, mfu: 17.04%\n",
      "epoch: 104/1000, loss: 2.2753, mfu: 16.37%\n",
      "epoch: 105/1000, loss: 2.2177, mfu: 16.84%\n",
      "epoch: 106/1000, loss: 2.2099, mfu: 17.23%\n",
      "epoch: 107/1000, loss: 2.2116, mfu: 17.17%\n",
      "epoch: 108/1000, loss: 2.2636, mfu: 17.20%\n",
      "epoch: 109/1000, loss: 2.2480, mfu: 16.15%\n",
      "epoch: 110/1000, loss: 2.2215, mfu: 16.88%\n",
      "epoch: 111/1000, loss: 2.2258, mfu: 17.73%\n",
      "epoch: 112/1000, loss: 2.2039, mfu: 15.53%\n",
      "epoch: 113/1000, loss: 2.2274, mfu: 17.80%\n",
      "epoch: 114/1000, loss: 2.2375, mfu: 17.28%\n",
      "epoch: 115/1000, loss: 2.1762, mfu: 18.18%\n",
      "epoch: 116/1000, loss: 2.2321, mfu: 16.66%\n",
      "epoch: 117/1000, loss: 2.1728, mfu: 18.44%\n",
      "epoch: 118/1000, loss: 2.1853, mfu: 15.72%\n",
      "epoch: 119/1000, loss: 2.1870, mfu: 17.35%\n",
      "epoch: 120/1000, loss: 2.1734, mfu: 17.56%\n",
      "epoch: 121/1000, loss: 2.1901, mfu: 16.79%\n",
      "epoch: 122/1000, loss: 2.1749, mfu: 16.88%\n",
      "epoch: 123/1000, loss: 2.1793, mfu: 17.92%\n",
      "epoch: 124/1000, loss: 2.1460, mfu: 16.66%\n",
      "epoch: 125/1000, loss: 2.1652, mfu: 16.69%\n",
      "epoch: 126/1000, loss: 2.1691, mfu: 17.79%\n",
      "epoch: 127/1000, loss: 2.1394, mfu: 17.39%\n",
      "epoch: 128/1000, loss: 2.1186, mfu: 16.21%\n",
      "epoch: 129/1000, loss: 2.1304, mfu: 16.34%\n",
      "epoch: 130/1000, loss: 2.1418, mfu: 17.74%\n",
      "epoch: 131/1000, loss: 2.1057, mfu: 17.89%\n",
      "epoch: 132/1000, loss: 2.1112, mfu: 17.16%\n",
      "epoch: 133/1000, loss: 2.1236, mfu: 16.48%\n",
      "epoch: 134/1000, loss: 2.0920, mfu: 15.56%\n",
      "epoch: 135/1000, loss: 2.1144, mfu: 14.54%\n",
      "epoch: 136/1000, loss: 2.0956, mfu: 16.73%\n",
      "epoch: 137/1000, loss: 2.0809, mfu: 16.94%\n",
      "epoch: 138/1000, loss: 2.1118, mfu: 15.98%\n",
      "epoch: 139/1000, loss: 2.1058, mfu: 15.55%\n",
      "epoch: 140/1000, loss: 2.0551, mfu: 15.55%\n",
      "epoch: 141/1000, loss: 2.0461, mfu: 16.66%\n",
      "epoch: 142/1000, loss: 2.0701, mfu: 17.57%\n",
      "epoch: 143/1000, loss: 2.0703, mfu: 19.20%\n",
      "epoch: 144/1000, loss: 2.0704, mfu: 16.48%\n",
      "epoch: 145/1000, loss: 2.0913, mfu: 15.69%\n",
      "epoch: 146/1000, loss: 2.0514, mfu: 16.95%\n",
      "epoch: 147/1000, loss: 2.0922, mfu: 16.88%\n",
      "epoch: 148/1000, loss: 2.0316, mfu: 16.23%\n",
      "epoch: 149/1000, loss: 2.0185, mfu: 16.72%\n",
      "epoch: 150/1000, loss: 2.0525, mfu: 13.96%\n",
      "epoch: 151/1000, loss: 2.0439, mfu: 17.06%\n",
      "epoch: 152/1000, loss: 2.0609, mfu: 15.96%\n",
      "epoch: 153/1000, loss: 2.0245, mfu: 16.58%\n",
      "epoch: 154/1000, loss: 2.0625, mfu: 17.67%\n",
      "epoch: 155/1000, loss: 2.0263, mfu: 18.38%\n",
      "epoch: 156/1000, loss: 1.9904, mfu: 15.52%\n",
      "epoch: 157/1000, loss: 2.0162, mfu: 17.18%\n",
      "epoch: 158/1000, loss: 2.0404, mfu: 14.81%\n",
      "epoch: 159/1000, loss: 2.0398, mfu: 16.99%\n",
      "epoch: 160/1000, loss: 1.9927, mfu: 17.36%\n",
      "epoch: 161/1000, loss: 1.9987, mfu: 15.97%\n",
      "epoch: 162/1000, loss: 1.9610, mfu: 16.08%\n",
      "epoch: 163/1000, loss: 2.0110, mfu: 17.46%\n",
      "epoch: 164/1000, loss: 1.9752, mfu: 17.59%\n",
      "epoch: 165/1000, loss: 1.9726, mfu: 17.14%\n",
      "epoch: 166/1000, loss: 1.9803, mfu: 17.25%\n",
      "epoch: 167/1000, loss: 2.0361, mfu: 18.15%\n",
      "epoch: 168/1000, loss: 1.9754, mfu: 16.79%\n",
      "epoch: 169/1000, loss: 2.0063, mfu: 17.51%\n",
      "epoch: 170/1000, loss: 1.9643, mfu: 17.23%\n",
      "epoch: 171/1000, loss: 1.9232, mfu: 17.32%\n",
      "epoch: 172/1000, loss: 1.9516, mfu: 16.96%\n",
      "epoch: 173/1000, loss: 1.9500, mfu: 16.49%\n",
      "epoch: 174/1000, loss: 1.9922, mfu: 15.63%\n",
      "epoch: 175/1000, loss: 1.9571, mfu: 16.21%\n",
      "epoch: 176/1000, loss: 1.9572, mfu: 17.64%\n",
      "epoch: 177/1000, loss: 1.9334, mfu: 16.34%\n",
      "epoch: 178/1000, loss: 1.9588, mfu: 16.44%\n",
      "epoch: 179/1000, loss: 1.9523, mfu: 15.41%\n",
      "epoch: 180/1000, loss: 1.9239, mfu: 15.60%\n",
      "epoch: 181/1000, loss: 1.9625, mfu: 15.56%\n",
      "epoch: 182/1000, loss: 1.9485, mfu: 18.05%\n",
      "epoch: 183/1000, loss: 1.9126, mfu: 17.35%\n",
      "epoch: 184/1000, loss: 1.9016, mfu: 16.47%\n",
      "epoch: 185/1000, loss: 1.9402, mfu: 15.71%\n",
      "epoch: 186/1000, loss: 1.9304, mfu: 16.48%\n",
      "epoch: 187/1000, loss: 1.9071, mfu: 15.23%\n",
      "epoch: 188/1000, loss: 1.9176, mfu: 14.93%\n",
      "epoch: 189/1000, loss: 1.8957, mfu: 16.08%\n",
      "epoch: 190/1000, loss: 1.8791, mfu: 17.36%\n",
      "epoch: 191/1000, loss: 1.9179, mfu: 16.90%\n",
      "epoch: 192/1000, loss: 1.9055, mfu: 17.71%\n",
      "epoch: 193/1000, loss: 1.8872, mfu: 16.44%\n",
      "epoch: 194/1000, loss: 1.9032, mfu: 18.11%\n",
      "epoch: 195/1000, loss: 1.8700, mfu: 17.40%\n",
      "epoch: 196/1000, loss: 1.9126, mfu: 16.45%\n",
      "epoch: 197/1000, loss: 1.8898, mfu: 16.31%\n",
      "epoch: 198/1000, loss: 1.8958, mfu: 17.35%\n",
      "epoch: 199/1000, loss: 1.8888, mfu: 16.44%\n",
      "epoch: 200/1000, loss: 1.8748, mfu: 17.16%\n",
      "epoch: 201/1000, loss: 1.8705, mfu: 15.86%\n",
      "epoch: 202/1000, loss: 1.8680, mfu: 15.70%\n",
      "epoch: 203/1000, loss: 1.8709, mfu: 16.61%\n",
      "epoch: 204/1000, loss: 1.8861, mfu: 16.05%\n",
      "epoch: 205/1000, loss: 1.8612, mfu: 15.45%\n",
      "epoch: 206/1000, loss: 1.8420, mfu: 13.22%\n",
      "epoch: 207/1000, loss: 1.8262, mfu: 15.21%\n",
      "epoch: 208/1000, loss: 1.8625, mfu: 16.41%\n",
      "epoch: 209/1000, loss: 1.8237, mfu: 14.91%\n",
      "epoch: 210/1000, loss: 1.8637, mfu: 16.50%\n",
      "epoch: 211/1000, loss: 1.8576, mfu: 17.88%\n",
      "epoch: 212/1000, loss: 1.8470, mfu: 15.66%\n",
      "epoch: 213/1000, loss: 1.8361, mfu: 18.04%\n",
      "epoch: 214/1000, loss: 1.8358, mfu: 16.31%\n",
      "epoch: 215/1000, loss: 1.8478, mfu: 16.79%\n",
      "epoch: 216/1000, loss: 1.8433, mfu: 15.48%\n",
      "epoch: 217/1000, loss: 1.8261, mfu: 16.88%\n",
      "epoch: 218/1000, loss: 1.8734, mfu: 14.04%\n",
      "epoch: 219/1000, loss: 1.8381, mfu: 17.78%\n",
      "epoch: 220/1000, loss: 1.8313, mfu: 16.95%\n",
      "epoch: 221/1000, loss: 1.8304, mfu: 16.25%\n",
      "epoch: 222/1000, loss: 1.8353, mfu: 16.89%\n",
      "epoch: 223/1000, loss: 1.7903, mfu: 16.41%\n",
      "epoch: 224/1000, loss: 1.7959, mfu: 17.97%\n",
      "epoch: 225/1000, loss: 1.7929, mfu: 17.08%\n",
      "epoch: 226/1000, loss: 1.7454, mfu: 17.07%\n",
      "epoch: 227/1000, loss: 1.8172, mfu: 16.00%\n",
      "epoch: 228/1000, loss: 1.7827, mfu: 16.82%\n",
      "epoch: 229/1000, loss: 1.7946, mfu: 16.95%\n",
      "epoch: 230/1000, loss: 1.7747, mfu: 15.63%\n",
      "epoch: 231/1000, loss: 1.7864, mfu: 15.32%\n",
      "epoch: 232/1000, loss: 1.8072, mfu: 16.22%\n",
      "epoch: 233/1000, loss: 1.7704, mfu: 16.93%\n",
      "epoch: 234/1000, loss: 1.7994, mfu: 17.34%\n",
      "epoch: 235/1000, loss: 1.7602, mfu: 14.77%\n",
      "epoch: 236/1000, loss: 1.7833, mfu: 16.82%\n",
      "epoch: 237/1000, loss: 1.7999, mfu: 15.88%\n",
      "epoch: 238/1000, loss: 1.7903, mfu: 16.37%\n",
      "epoch: 239/1000, loss: 1.7924, mfu: 16.19%\n",
      "epoch: 240/1000, loss: 1.7874, mfu: 16.14%\n",
      "epoch: 241/1000, loss: 1.7843, mfu: 17.17%\n",
      "epoch: 242/1000, loss: 1.7722, mfu: 16.64%\n",
      "epoch: 243/1000, loss: 1.7587, mfu: 16.86%\n",
      "epoch: 244/1000, loss: 1.7839, mfu: 16.42%\n",
      "epoch: 245/1000, loss: 1.7200, mfu: 17.02%\n",
      "epoch: 246/1000, loss: 1.7592, mfu: 16.71%\n",
      "epoch: 247/1000, loss: 1.7979, mfu: 17.24%\n",
      "epoch: 248/1000, loss: 1.7421, mfu: 17.60%\n",
      "epoch: 249/1000, loss: 1.7731, mfu: 13.74%\n",
      "epoch: 250/1000, loss: 1.7579, mfu: 15.87%\n",
      "epoch: 251/1000, loss: 1.7346, mfu: 17.92%\n",
      "epoch: 252/1000, loss: 1.7699, mfu: 16.45%\n",
      "epoch: 253/1000, loss: 1.7212, mfu: 18.44%\n",
      "epoch: 254/1000, loss: 1.7364, mfu: 16.68%\n",
      "epoch: 255/1000, loss: 1.7609, mfu: 17.54%\n",
      "epoch: 256/1000, loss: 1.7070, mfu: 17.70%\n",
      "epoch: 257/1000, loss: 1.7269, mfu: 16.94%\n",
      "epoch: 258/1000, loss: 1.7471, mfu: 18.18%\n",
      "epoch: 259/1000, loss: 1.7191, mfu: 16.25%\n",
      "epoch: 260/1000, loss: 1.7417, mfu: 15.07%\n",
      "epoch: 261/1000, loss: 1.7149, mfu: 17.26%\n",
      "epoch: 262/1000, loss: 1.7269, mfu: 17.17%\n",
      "epoch: 263/1000, loss: 1.7330, mfu: 16.21%\n",
      "epoch: 264/1000, loss: 1.7463, mfu: 16.82%\n",
      "epoch: 265/1000, loss: 1.6994, mfu: 16.13%\n",
      "epoch: 266/1000, loss: 1.7053, mfu: 17.63%\n",
      "epoch: 267/1000, loss: 1.7568, mfu: 16.90%\n",
      "epoch: 268/1000, loss: 1.7271, mfu: 17.25%\n",
      "epoch: 269/1000, loss: 1.7446, mfu: 15.57%\n",
      "epoch: 270/1000, loss: 1.7467, mfu: 16.26%\n",
      "epoch: 271/1000, loss: 1.6858, mfu: 16.80%\n",
      "epoch: 272/1000, loss: 1.6875, mfu: 16.86%\n",
      "epoch: 273/1000, loss: 1.6687, mfu: 17.70%\n",
      "epoch: 274/1000, loss: 1.6922, mfu: 15.35%\n",
      "epoch: 275/1000, loss: 1.7139, mfu: 16.65%\n",
      "epoch: 276/1000, loss: 1.6960, mfu: 16.01%\n",
      "epoch: 277/1000, loss: 1.7379, mfu: 15.84%\n",
      "epoch: 278/1000, loss: 1.6677, mfu: 16.52%\n",
      "epoch: 279/1000, loss: 1.7198, mfu: 16.78%\n",
      "epoch: 280/1000, loss: 1.7003, mfu: 18.08%\n",
      "epoch: 281/1000, loss: 1.7381, mfu: 16.40%\n",
      "epoch: 282/1000, loss: 1.7313, mfu: 17.77%\n",
      "epoch: 283/1000, loss: 1.6802, mfu: 16.50%\n",
      "epoch: 284/1000, loss: 1.6861, mfu: 17.22%\n",
      "epoch: 285/1000, loss: 1.6787, mfu: 17.15%\n",
      "epoch: 286/1000, loss: 1.6894, mfu: 17.43%\n",
      "epoch: 287/1000, loss: 1.6330, mfu: 17.65%\n",
      "epoch: 288/1000, loss: 1.6815, mfu: 14.95%\n",
      "epoch: 289/1000, loss: 1.6562, mfu: 16.29%\n",
      "epoch: 290/1000, loss: 1.6929, mfu: 14.72%\n",
      "epoch: 291/1000, loss: 1.6709, mfu: 16.60%\n",
      "epoch: 292/1000, loss: 1.6889, mfu: 16.33%\n",
      "epoch: 293/1000, loss: 1.6468, mfu: 16.51%\n",
      "epoch: 294/1000, loss: 1.6210, mfu: 15.88%\n",
      "epoch: 295/1000, loss: 1.6577, mfu: 17.56%\n",
      "epoch: 296/1000, loss: 1.7262, mfu: 15.90%\n",
      "epoch: 297/1000, loss: 1.6602, mfu: 16.95%\n",
      "epoch: 298/1000, loss: 1.6629, mfu: 16.71%\n",
      "epoch: 299/1000, loss: 1.6657, mfu: 17.56%\n",
      "epoch: 300/1000, loss: 1.6899, mfu: 15.74%\n",
      "epoch: 301/1000, loss: 1.6447, mfu: 16.28%\n",
      "epoch: 302/1000, loss: 1.6301, mfu: 16.05%\n",
      "epoch: 303/1000, loss: 1.6774, mfu: 16.29%\n",
      "epoch: 304/1000, loss: 1.6423, mfu: 15.84%\n",
      "epoch: 305/1000, loss: 1.6688, mfu: 17.72%\n",
      "epoch: 306/1000, loss: 1.6410, mfu: 16.98%\n",
      "epoch: 307/1000, loss: 1.6646, mfu: 15.64%\n",
      "epoch: 308/1000, loss: 1.6439, mfu: 15.92%\n",
      "epoch: 309/1000, loss: 1.6198, mfu: 16.93%\n",
      "epoch: 310/1000, loss: 1.6388, mfu: 16.12%\n",
      "epoch: 311/1000, loss: 1.6624, mfu: 17.07%\n",
      "epoch: 312/1000, loss: 1.6306, mfu: 16.34%\n",
      "epoch: 313/1000, loss: 1.6276, mfu: 17.29%\n",
      "epoch: 314/1000, loss: 1.6007, mfu: 17.74%\n",
      "epoch: 315/1000, loss: 1.6569, mfu: 14.36%\n",
      "epoch: 316/1000, loss: 1.5952, mfu: 14.50%\n",
      "epoch: 317/1000, loss: 1.6246, mfu: 15.59%\n",
      "epoch: 318/1000, loss: 1.6598, mfu: 16.45%\n",
      "epoch: 319/1000, loss: 1.6613, mfu: 16.97%\n",
      "epoch: 320/1000, loss: 1.5890, mfu: 15.94%\n",
      "epoch: 321/1000, loss: 1.6272, mfu: 16.17%\n",
      "epoch: 322/1000, loss: 1.6222, mfu: 16.07%\n",
      "epoch: 323/1000, loss: 1.6039, mfu: 16.86%\n",
      "epoch: 324/1000, loss: 1.6039, mfu: 14.50%\n",
      "epoch: 325/1000, loss: 1.6278, mfu: 17.03%\n",
      "epoch: 326/1000, loss: 1.6097, mfu: 14.92%\n",
      "epoch: 327/1000, loss: 1.6373, mfu: 14.73%\n",
      "epoch: 328/1000, loss: 1.5886, mfu: 17.45%\n",
      "epoch: 329/1000, loss: 1.6203, mfu: 15.22%\n",
      "epoch: 330/1000, loss: 1.5997, mfu: 15.23%\n",
      "epoch: 331/1000, loss: 1.6145, mfu: 16.72%\n",
      "epoch: 332/1000, loss: 1.6327, mfu: 17.38%\n",
      "epoch: 333/1000, loss: 1.5698, mfu: 16.06%\n",
      "epoch: 334/1000, loss: 1.5795, mfu: 16.10%\n",
      "epoch: 335/1000, loss: 1.5643, mfu: 16.61%\n",
      "epoch: 336/1000, loss: 1.6066, mfu: 17.75%\n",
      "epoch: 337/1000, loss: 1.5907, mfu: 16.85%\n",
      "epoch: 338/1000, loss: 1.5823, mfu: 17.67%\n",
      "epoch: 339/1000, loss: 1.5521, mfu: 17.29%\n",
      "epoch: 340/1000, loss: 1.5784, mfu: 17.26%\n",
      "epoch: 341/1000, loss: 1.6108, mfu: 16.68%\n",
      "epoch: 342/1000, loss: 1.5903, mfu: 16.84%\n",
      "epoch: 343/1000, loss: 1.5673, mfu: 16.94%\n",
      "epoch: 344/1000, loss: 1.5755, mfu: 17.12%\n",
      "epoch: 345/1000, loss: 1.5699, mfu: 16.31%\n",
      "epoch: 346/1000, loss: 1.6343, mfu: 16.46%\n",
      "epoch: 347/1000, loss: 1.5793, mfu: 17.44%\n",
      "epoch: 348/1000, loss: 1.5638, mfu: 18.14%\n",
      "epoch: 349/1000, loss: 1.5768, mfu: 18.24%\n",
      "epoch: 350/1000, loss: 1.5721, mfu: 15.36%\n",
      "epoch: 351/1000, loss: 1.5564, mfu: 14.89%\n",
      "epoch: 352/1000, loss: 1.5762, mfu: 17.86%\n",
      "epoch: 353/1000, loss: 1.5982, mfu: 18.14%\n",
      "epoch: 354/1000, loss: 1.5478, mfu: 16.68%\n",
      "epoch: 355/1000, loss: 1.5892, mfu: 18.25%\n",
      "epoch: 356/1000, loss: 1.5595, mfu: 17.10%\n",
      "epoch: 357/1000, loss: 1.5596, mfu: 16.87%\n",
      "epoch: 358/1000, loss: 1.5967, mfu: 16.03%\n",
      "epoch: 359/1000, loss: 1.5731, mfu: 16.91%\n",
      "epoch: 360/1000, loss: 1.5479, mfu: 12.59%\n",
      "epoch: 361/1000, loss: 1.5477, mfu: 17.47%\n",
      "epoch: 362/1000, loss: 1.5762, mfu: 17.38%\n",
      "epoch: 363/1000, loss: 1.5216, mfu: 15.44%\n",
      "epoch: 364/1000, loss: 1.5436, mfu: 16.37%\n",
      "epoch: 365/1000, loss: 1.5659, mfu: 15.69%\n",
      "epoch: 366/1000, loss: 1.5990, mfu: 15.30%\n",
      "epoch: 367/1000, loss: 1.5313, mfu: 16.43%\n",
      "epoch: 368/1000, loss: 1.5303, mfu: 15.98%\n",
      "epoch: 369/1000, loss: 1.5587, mfu: 14.41%\n",
      "epoch: 370/1000, loss: 1.5304, mfu: 15.59%\n",
      "epoch: 371/1000, loss: 1.5177, mfu: 15.16%\n",
      "epoch: 372/1000, loss: 1.5678, mfu: 15.28%\n",
      "epoch: 373/1000, loss: 1.5355, mfu: 15.10%\n",
      "epoch: 374/1000, loss: 1.5334, mfu: 15.88%\n",
      "epoch: 375/1000, loss: 1.4930, mfu: 15.25%\n",
      "epoch: 376/1000, loss: 1.5774, mfu: 15.88%\n",
      "epoch: 377/1000, loss: 1.4908, mfu: 16.07%\n",
      "epoch: 378/1000, loss: 1.4967, mfu: 15.98%\n",
      "epoch: 379/1000, loss: 1.5110, mfu: 17.14%\n",
      "epoch: 380/1000, loss: 1.5832, mfu: 16.41%\n",
      "epoch: 381/1000, loss: 1.5500, mfu: 15.37%\n",
      "epoch: 382/1000, loss: 1.4936, mfu: 16.52%\n",
      "epoch: 383/1000, loss: 1.5343, mfu: 16.81%\n",
      "epoch: 384/1000, loss: 1.5103, mfu: 16.67%\n",
      "epoch: 385/1000, loss: 1.4929, mfu: 17.30%\n",
      "epoch: 386/1000, loss: 1.5451, mfu: 17.62%\n",
      "epoch: 387/1000, loss: 1.5079, mfu: 17.69%\n",
      "epoch: 388/1000, loss: 1.5278, mfu: 16.52%\n",
      "epoch: 389/1000, loss: 1.5127, mfu: 16.33%\n",
      "epoch: 390/1000, loss: 1.4965, mfu: 16.60%\n",
      "epoch: 391/1000, loss: 1.5576, mfu: 16.15%\n",
      "epoch: 392/1000, loss: 1.5140, mfu: 17.05%\n",
      "epoch: 393/1000, loss: 1.5071, mfu: 16.19%\n",
      "epoch: 394/1000, loss: 1.5019, mfu: 17.42%\n",
      "epoch: 395/1000, loss: 1.5178, mfu: 16.31%\n",
      "epoch: 396/1000, loss: 1.4959, mfu: 17.69%\n",
      "epoch: 397/1000, loss: 1.5240, mfu: 16.46%\n",
      "epoch: 398/1000, loss: 1.5056, mfu: 17.23%\n",
      "epoch: 399/1000, loss: 1.4858, mfu: 16.02%\n",
      "epoch: 400/1000, loss: 1.4935, mfu: 17.29%\n",
      "epoch: 401/1000, loss: 1.5306, mfu: 13.23%\n",
      "epoch: 402/1000, loss: 1.5286, mfu: 13.70%\n",
      "epoch: 403/1000, loss: 1.5143, mfu: 15.74%\n",
      "epoch: 404/1000, loss: 1.4670, mfu: 16.69%\n",
      "epoch: 405/1000, loss: 1.4634, mfu: 16.99%\n",
      "epoch: 406/1000, loss: 1.5193, mfu: 15.93%\n",
      "epoch: 407/1000, loss: 1.4721, mfu: 16.00%\n",
      "epoch: 408/1000, loss: 1.5147, mfu: 17.42%\n",
      "epoch: 409/1000, loss: 1.4544, mfu: 17.45%\n",
      "epoch: 410/1000, loss: 1.4995, mfu: 17.96%\n",
      "epoch: 411/1000, loss: 1.4683, mfu: 15.65%\n",
      "epoch: 412/1000, loss: 1.5328, mfu: 15.81%\n",
      "epoch: 413/1000, loss: 1.4890, mfu: 16.21%\n",
      "epoch: 414/1000, loss: 1.5265, mfu: 16.51%\n",
      "epoch: 415/1000, loss: 1.4983, mfu: 17.46%\n",
      "epoch: 416/1000, loss: 1.4783, mfu: 15.16%\n",
      "epoch: 417/1000, loss: 1.5058, mfu: 15.34%\n",
      "epoch: 418/1000, loss: 1.4863, mfu: 16.27%\n",
      "epoch: 419/1000, loss: 1.5411, mfu: 15.37%\n",
      "epoch: 420/1000, loss: 1.4888, mfu: 17.17%\n",
      "epoch: 421/1000, loss: 1.4595, mfu: 16.36%\n",
      "epoch: 422/1000, loss: 1.4715, mfu: 15.45%\n",
      "epoch: 423/1000, loss: 1.4264, mfu: 17.98%\n",
      "epoch: 424/1000, loss: 1.4851, mfu: 17.36%\n",
      "epoch: 425/1000, loss: 1.5074, mfu: 17.05%\n",
      "epoch: 426/1000, loss: 1.4365, mfu: 17.22%\n",
      "epoch: 427/1000, loss: 1.4922, mfu: 16.99%\n",
      "epoch: 428/1000, loss: 1.3987, mfu: 16.62%\n",
      "epoch: 429/1000, loss: 1.5020, mfu: 16.47%\n",
      "epoch: 430/1000, loss: 1.4971, mfu: 17.50%\n",
      "epoch: 431/1000, loss: 1.4475, mfu: 17.07%\n",
      "epoch: 432/1000, loss: 1.4614, mfu: 16.23%\n",
      "epoch: 433/1000, loss: 1.4425, mfu: 16.00%\n",
      "epoch: 434/1000, loss: 1.4385, mfu: 18.08%\n",
      "epoch: 435/1000, loss: 1.4373, mfu: 16.56%\n",
      "epoch: 436/1000, loss: 1.4431, mfu: 15.16%\n",
      "epoch: 437/1000, loss: 1.4052, mfu: 17.36%\n",
      "epoch: 438/1000, loss: 1.4151, mfu: 17.11%\n",
      "epoch: 439/1000, loss: 1.4551, mfu: 16.55%\n",
      "epoch: 440/1000, loss: 1.4716, mfu: 17.10%\n",
      "epoch: 441/1000, loss: 1.4352, mfu: 16.81%\n",
      "epoch: 442/1000, loss: 1.4723, mfu: 15.73%\n",
      "epoch: 443/1000, loss: 1.3937, mfu: 17.27%\n",
      "epoch: 444/1000, loss: 1.4410, mfu: 16.06%\n",
      "epoch: 445/1000, loss: 1.3815, mfu: 17.07%\n",
      "epoch: 446/1000, loss: 1.4206, mfu: 17.29%\n",
      "epoch: 447/1000, loss: 1.4974, mfu: 16.11%\n",
      "epoch: 448/1000, loss: 1.4423, mfu: 17.18%\n",
      "epoch: 449/1000, loss: 1.4081, mfu: 16.01%\n",
      "epoch: 450/1000, loss: 1.4334, mfu: 16.11%\n",
      "epoch: 451/1000, loss: 1.4169, mfu: 14.93%\n",
      "epoch: 452/1000, loss: 1.4732, mfu: 17.06%\n",
      "epoch: 453/1000, loss: 1.4473, mfu: 17.70%\n",
      "epoch: 454/1000, loss: 1.4846, mfu: 16.21%\n",
      "epoch: 455/1000, loss: 1.3978, mfu: 16.50%\n",
      "epoch: 456/1000, loss: 1.4198, mfu: 17.41%\n",
      "epoch: 457/1000, loss: 1.4847, mfu: 17.02%\n",
      "epoch: 458/1000, loss: 1.4424, mfu: 16.96%\n",
      "epoch: 459/1000, loss: 1.4159, mfu: 16.79%\n",
      "epoch: 460/1000, loss: 1.4052, mfu: 16.02%\n",
      "epoch: 461/1000, loss: 1.4400, mfu: 17.36%\n",
      "epoch: 462/1000, loss: 1.3947, mfu: 15.99%\n",
      "epoch: 463/1000, loss: 1.4534, mfu: 16.90%\n",
      "epoch: 464/1000, loss: 1.4746, mfu: 17.79%\n",
      "epoch: 465/1000, loss: 1.4136, mfu: 15.82%\n",
      "epoch: 466/1000, loss: 1.3964, mfu: 17.15%\n",
      "epoch: 467/1000, loss: 1.3880, mfu: 17.20%\n",
      "epoch: 468/1000, loss: 1.3931, mfu: 9.95%\n",
      "epoch: 469/1000, loss: 1.4448, mfu: 8.12%\n",
      "epoch: 470/1000, loss: 1.4679, mfu: 7.79%\n",
      "epoch: 471/1000, loss: 1.4300, mfu: 16.50%\n",
      "epoch: 472/1000, loss: 1.3608, mfu: 8.15%\n",
      "epoch: 473/1000, loss: 1.4068, mfu: 15.60%\n",
      "epoch: 474/1000, loss: 1.3799, mfu: 15.34%\n",
      "epoch: 475/1000, loss: 1.4517, mfu: 15.79%\n",
      "epoch: 476/1000, loss: 1.4356, mfu: 13.75%\n",
      "epoch: 477/1000, loss: 1.4010, mfu: 14.27%\n",
      "epoch: 478/1000, loss: 1.4201, mfu: 14.87%\n",
      "epoch: 479/1000, loss: 1.3926, mfu: 13.30%\n",
      "epoch: 480/1000, loss: 1.3564, mfu: 14.92%\n",
      "epoch: 481/1000, loss: 1.4141, mfu: 15.63%\n",
      "epoch: 482/1000, loss: 1.3989, mfu: 15.42%\n",
      "epoch: 483/1000, loss: 1.4128, mfu: 15.01%\n",
      "epoch: 484/1000, loss: 1.4446, mfu: 15.83%\n",
      "epoch: 485/1000, loss: 1.4435, mfu: 15.77%\n",
      "epoch: 486/1000, loss: 1.3914, mfu: 12.71%\n",
      "epoch: 487/1000, loss: 1.4096, mfu: 15.21%\n",
      "epoch: 488/1000, loss: 1.3913, mfu: 16.10%\n",
      "epoch: 489/1000, loss: 1.3802, mfu: 16.46%\n",
      "epoch: 490/1000, loss: 1.3836, mfu: 15.89%\n",
      "epoch: 491/1000, loss: 1.4399, mfu: 16.50%\n",
      "epoch: 492/1000, loss: 1.4163, mfu: 10.53%\n",
      "epoch: 493/1000, loss: 1.3776, mfu: 17.12%\n",
      "epoch: 494/1000, loss: 1.4019, mfu: 14.88%\n",
      "epoch: 495/1000, loss: 1.4190, mfu: 16.24%\n",
      "epoch: 496/1000, loss: 1.3573, mfu: 13.46%\n",
      "epoch: 497/1000, loss: 1.3981, mfu: 12.95%\n",
      "epoch: 498/1000, loss: 1.3892, mfu: 16.14%\n",
      "epoch: 499/1000, loss: 1.3806, mfu: 12.71%\n",
      "epoch: 500/1000, loss: 1.4401, mfu: 15.97%\n",
      "epoch: 501/1000, loss: 1.3962, mfu: 14.31%\n",
      "epoch: 502/1000, loss: 1.4196, mfu: 16.63%\n",
      "epoch: 503/1000, loss: 1.3625, mfu: 17.00%\n",
      "epoch: 504/1000, loss: 1.4361, mfu: 14.39%\n",
      "epoch: 505/1000, loss: 1.4058, mfu: 16.38%\n",
      "epoch: 506/1000, loss: 1.3817, mfu: 16.68%\n",
      "epoch: 507/1000, loss: 1.3749, mfu: 15.96%\n",
      "epoch: 508/1000, loss: 1.3846, mfu: 16.42%\n",
      "epoch: 509/1000, loss: 1.3958, mfu: 15.43%\n",
      "epoch: 510/1000, loss: 1.4206, mfu: 17.65%\n",
      "epoch: 511/1000, loss: 1.4351, mfu: 17.47%\n",
      "epoch: 512/1000, loss: 1.3918, mfu: 15.22%\n",
      "epoch: 513/1000, loss: 1.3893, mfu: 13.88%\n",
      "epoch: 514/1000, loss: 1.3796, mfu: 14.43%\n",
      "epoch: 515/1000, loss: 1.3369, mfu: 15.08%\n",
      "epoch: 516/1000, loss: 1.3730, mfu: 15.06%\n",
      "epoch: 517/1000, loss: 1.4084, mfu: 16.63%\n",
      "epoch: 518/1000, loss: 1.3595, mfu: 16.04%\n",
      "epoch: 519/1000, loss: 1.3662, mfu: 16.62%\n",
      "epoch: 520/1000, loss: 1.3824, mfu: 16.08%\n",
      "epoch: 521/1000, loss: 1.3851, mfu: 15.89%\n",
      "epoch: 522/1000, loss: 1.3358, mfu: 17.28%\n",
      "epoch: 523/1000, loss: 1.3835, mfu: 15.21%\n",
      "epoch: 524/1000, loss: 1.3733, mfu: 16.06%\n",
      "epoch: 525/1000, loss: 1.3448, mfu: 16.69%\n",
      "epoch: 526/1000, loss: 1.3127, mfu: 15.79%\n",
      "epoch: 527/1000, loss: 1.3430, mfu: 17.22%\n",
      "epoch: 528/1000, loss: 1.3533, mfu: 16.27%\n",
      "epoch: 529/1000, loss: 1.3696, mfu: 15.91%\n",
      "epoch: 530/1000, loss: 1.3637, mfu: 15.89%\n",
      "epoch: 531/1000, loss: 1.4269, mfu: 15.70%\n",
      "epoch: 532/1000, loss: 1.3372, mfu: 15.53%\n",
      "epoch: 533/1000, loss: 1.4055, mfu: 17.43%\n",
      "epoch: 534/1000, loss: 1.3869, mfu: 16.48%\n",
      "epoch: 535/1000, loss: 1.3541, mfu: 17.31%\n",
      "epoch: 536/1000, loss: 1.4064, mfu: 16.93%\n",
      "epoch: 537/1000, loss: 1.3362, mfu: 16.25%\n",
      "epoch: 538/1000, loss: 1.3766, mfu: 16.13%\n",
      "epoch: 539/1000, loss: 1.3981, mfu: 15.45%\n",
      "epoch: 540/1000, loss: 1.3603, mfu: 17.24%\n",
      "epoch: 541/1000, loss: 1.3621, mfu: 14.66%\n",
      "epoch: 542/1000, loss: 1.3694, mfu: 16.20%\n",
      "epoch: 543/1000, loss: 1.3621, mfu: 17.30%\n",
      "epoch: 544/1000, loss: 1.4032, mfu: 17.46%\n",
      "epoch: 545/1000, loss: 1.3126, mfu: 14.67%\n",
      "epoch: 546/1000, loss: 1.3528, mfu: 15.61%\n",
      "epoch: 547/1000, loss: 1.3524, mfu: 16.23%\n",
      "epoch: 548/1000, loss: 1.3367, mfu: 16.54%\n",
      "epoch: 549/1000, loss: 1.3404, mfu: 17.65%\n",
      "epoch: 550/1000, loss: 1.3877, mfu: 16.48%\n",
      "epoch: 551/1000, loss: 1.3139, mfu: 15.87%\n",
      "epoch: 552/1000, loss: 1.3389, mfu: 15.75%\n",
      "epoch: 553/1000, loss: 1.3739, mfu: 13.26%\n",
      "epoch: 554/1000, loss: 1.3219, mfu: 15.73%\n",
      "epoch: 555/1000, loss: 1.2964, mfu: 14.76%\n",
      "epoch: 556/1000, loss: 1.3206, mfu: 16.24%\n",
      "epoch: 557/1000, loss: 1.3008, mfu: 17.27%\n",
      "epoch: 558/1000, loss: 1.3332, mfu: 16.38%\n",
      "epoch: 559/1000, loss: 1.3640, mfu: 15.55%\n",
      "epoch: 560/1000, loss: 1.3354, mfu: 14.89%\n",
      "epoch: 561/1000, loss: 1.3032, mfu: 16.33%\n",
      "epoch: 562/1000, loss: 1.3112, mfu: 16.57%\n",
      "epoch: 563/1000, loss: 1.3216, mfu: 17.56%\n",
      "epoch: 564/1000, loss: 1.4037, mfu: 17.09%\n",
      "epoch: 565/1000, loss: 1.3769, mfu: 15.62%\n",
      "epoch: 566/1000, loss: 1.4112, mfu: 16.76%\n",
      "epoch: 567/1000, loss: 1.3736, mfu: 16.69%\n",
      "epoch: 568/1000, loss: 1.3568, mfu: 14.86%\n",
      "epoch: 569/1000, loss: 1.3265, mfu: 17.53%\n",
      "epoch: 570/1000, loss: 1.3132, mfu: 15.59%\n",
      "epoch: 571/1000, loss: 1.3540, mfu: 16.95%\n",
      "epoch: 572/1000, loss: 1.3308, mfu: 17.35%\n",
      "epoch: 573/1000, loss: 1.4135, mfu: 15.78%\n",
      "epoch: 574/1000, loss: 1.3567, mfu: 15.66%\n",
      "epoch: 575/1000, loss: 1.3342, mfu: 14.59%\n",
      "epoch: 576/1000, loss: 1.3614, mfu: 16.06%\n",
      "epoch: 577/1000, loss: 1.2920, mfu: 16.62%\n",
      "epoch: 578/1000, loss: 1.3218, mfu: 16.17%\n",
      "epoch: 579/1000, loss: 1.3008, mfu: 14.80%\n",
      "epoch: 580/1000, loss: 1.3032, mfu: 14.35%\n",
      "epoch: 581/1000, loss: 1.3400, mfu: 15.91%\n",
      "epoch: 582/1000, loss: 1.3176, mfu: 15.35%\n",
      "epoch: 583/1000, loss: 1.3427, mfu: 15.49%\n",
      "epoch: 584/1000, loss: 1.3210, mfu: 15.63%\n",
      "epoch: 585/1000, loss: 1.3481, mfu: 16.10%\n",
      "epoch: 586/1000, loss: 1.3591, mfu: 15.10%\n",
      "epoch: 587/1000, loss: 1.2727, mfu: 17.29%\n",
      "epoch: 588/1000, loss: 1.3118, mfu: 12.00%\n",
      "epoch: 589/1000, loss: 1.3054, mfu: 15.49%\n",
      "epoch: 590/1000, loss: 1.3176, mfu: 15.18%\n",
      "epoch: 591/1000, loss: 1.2758, mfu: 15.09%\n",
      "epoch: 592/1000, loss: 1.2899, mfu: 15.55%\n",
      "epoch: 593/1000, loss: 1.3146, mfu: 17.09%\n",
      "epoch: 594/1000, loss: 1.2863, mfu: 13.92%\n",
      "epoch: 595/1000, loss: 1.3299, mfu: 16.97%\n",
      "epoch: 596/1000, loss: 1.2918, mfu: 16.81%\n",
      "epoch: 597/1000, loss: 1.3021, mfu: 16.81%\n",
      "epoch: 598/1000, loss: 1.3342, mfu: 15.90%\n",
      "epoch: 599/1000, loss: 1.3336, mfu: 16.16%\n",
      "epoch: 600/1000, loss: 1.3386, mfu: 16.70%\n",
      "epoch: 601/1000, loss: 1.3106, mfu: 17.03%\n",
      "epoch: 602/1000, loss: 1.2546, mfu: 16.14%\n",
      "epoch: 603/1000, loss: 1.3538, mfu: 15.24%\n",
      "epoch: 604/1000, loss: 1.3485, mfu: 16.06%\n",
      "epoch: 605/1000, loss: 1.2382, mfu: 17.52%\n",
      "epoch: 606/1000, loss: 1.2984, mfu: 15.56%\n",
      "epoch: 607/1000, loss: 1.3159, mfu: 17.20%\n",
      "epoch: 608/1000, loss: 1.3040, mfu: 16.20%\n",
      "epoch: 609/1000, loss: 1.2862, mfu: 17.76%\n",
      "epoch: 610/1000, loss: 1.3150, mfu: 16.37%\n",
      "epoch: 611/1000, loss: 1.3076, mfu: 14.06%\n",
      "epoch: 612/1000, loss: 1.2931, mfu: 15.99%\n",
      "epoch: 613/1000, loss: 1.2812, mfu: 16.73%\n",
      "epoch: 614/1000, loss: 1.2875, mfu: 16.14%\n",
      "epoch: 615/1000, loss: 1.2780, mfu: 16.33%\n",
      "epoch: 616/1000, loss: 1.2748, mfu: 15.34%\n",
      "epoch: 617/1000, loss: 1.3292, mfu: 14.83%\n",
      "epoch: 618/1000, loss: 1.3037, mfu: 16.87%\n",
      "epoch: 619/1000, loss: 1.2788, mfu: 16.33%\n",
      "epoch: 620/1000, loss: 1.2789, mfu: 15.45%\n",
      "epoch: 621/1000, loss: 1.2778, mfu: 16.48%\n",
      "epoch: 622/1000, loss: 1.2802, mfu: 14.63%\n",
      "epoch: 623/1000, loss: 1.3468, mfu: 16.81%\n",
      "epoch: 624/1000, loss: 1.3228, mfu: 13.03%\n",
      "epoch: 625/1000, loss: 1.3275, mfu: 13.22%\n",
      "epoch: 626/1000, loss: 1.2530, mfu: 16.28%\n",
      "epoch: 627/1000, loss: 1.3160, mfu: 15.69%\n",
      "epoch: 628/1000, loss: 1.2546, mfu: 15.03%\n",
      "epoch: 629/1000, loss: 1.2455, mfu: 15.01%\n",
      "epoch: 630/1000, loss: 1.2673, mfu: 15.84%\n",
      "epoch: 631/1000, loss: 1.2886, mfu: 15.41%\n",
      "epoch: 632/1000, loss: 1.2889, mfu: 15.54%\n",
      "epoch: 633/1000, loss: 1.2737, mfu: 15.59%\n",
      "epoch: 634/1000, loss: 1.2949, mfu: 16.22%\n",
      "epoch: 635/1000, loss: 1.3080, mfu: 16.56%\n",
      "epoch: 636/1000, loss: 1.3063, mfu: 16.53%\n",
      "epoch: 637/1000, loss: 1.2777, mfu: 16.20%\n",
      "epoch: 638/1000, loss: 1.2736, mfu: 14.31%\n",
      "epoch: 639/1000, loss: 1.2401, mfu: 14.94%\n",
      "epoch: 640/1000, loss: 1.2713, mfu: 15.58%\n",
      "epoch: 641/1000, loss: 1.2960, mfu: 15.43%\n",
      "epoch: 642/1000, loss: 1.2621, mfu: 14.66%\n",
      "epoch: 643/1000, loss: 1.2624, mfu: 14.46%\n",
      "epoch: 644/1000, loss: 1.3012, mfu: 15.62%\n",
      "epoch: 645/1000, loss: 1.2317, mfu: 15.61%\n",
      "epoch: 646/1000, loss: 1.2999, mfu: 14.20%\n",
      "epoch: 647/1000, loss: 1.2855, mfu: 13.26%\n",
      "epoch: 648/1000, loss: 1.2817, mfu: 11.34%\n",
      "epoch: 649/1000, loss: 1.2140, mfu: 9.91%\n",
      "epoch: 650/1000, loss: 1.3013, mfu: 12.47%\n",
      "epoch: 651/1000, loss: 1.3057, mfu: 14.98%\n",
      "epoch: 652/1000, loss: 1.3103, mfu: 13.79%\n",
      "epoch: 653/1000, loss: 1.2552, mfu: 14.67%\n",
      "epoch: 654/1000, loss: 1.2577, mfu: 15.87%\n",
      "epoch: 655/1000, loss: 1.2666, mfu: 12.19%\n",
      "epoch: 656/1000, loss: 1.2584, mfu: 12.76%\n",
      "epoch: 657/1000, loss: 1.2628, mfu: 14.02%\n",
      "epoch: 658/1000, loss: 1.2574, mfu: 13.24%\n",
      "epoch: 659/1000, loss: 1.2205, mfu: 16.39%\n",
      "epoch: 660/1000, loss: 1.2627, mfu: 16.05%\n",
      "epoch: 661/1000, loss: 1.2902, mfu: 15.56%\n",
      "epoch: 662/1000, loss: 1.3156, mfu: 9.37%\n",
      "epoch: 663/1000, loss: 1.2529, mfu: 14.06%\n",
      "epoch: 664/1000, loss: 1.2773, mfu: 16.06%\n",
      "epoch: 665/1000, loss: 1.2473, mfu: 16.35%\n",
      "epoch: 666/1000, loss: 1.2321, mfu: 14.18%\n",
      "epoch: 667/1000, loss: 1.2425, mfu: 15.95%\n",
      "epoch: 668/1000, loss: 1.2201, mfu: 15.33%\n",
      "epoch: 669/1000, loss: 1.2581, mfu: 14.89%\n",
      "epoch: 670/1000, loss: 1.2604, mfu: 15.34%\n",
      "epoch: 671/1000, loss: 1.2338, mfu: 12.20%\n",
      "epoch: 672/1000, loss: 1.2877, mfu: 14.62%\n",
      "epoch: 673/1000, loss: 1.2490, mfu: 14.83%\n",
      "epoch: 674/1000, loss: 1.2360, mfu: 13.96%\n",
      "epoch: 675/1000, loss: 1.2009, mfu: 16.24%\n",
      "epoch: 676/1000, loss: 1.2213, mfu: 11.54%\n",
      "epoch: 677/1000, loss: 1.2488, mfu: 7.44%\n",
      "epoch: 678/1000, loss: 1.2003, mfu: 15.10%\n",
      "epoch: 679/1000, loss: 1.2215, mfu: 15.68%\n",
      "epoch: 680/1000, loss: 1.2479, mfu: 14.09%\n",
      "epoch: 681/1000, loss: 1.2271, mfu: 10.82%\n",
      "epoch: 682/1000, loss: 1.2014, mfu: 13.46%\n",
      "epoch: 683/1000, loss: 1.2666, mfu: 11.69%\n",
      "epoch: 684/1000, loss: 1.2460, mfu: 15.19%\n",
      "epoch: 685/1000, loss: 1.2919, mfu: 14.22%\n",
      "epoch: 686/1000, loss: 1.2518, mfu: 16.43%\n",
      "epoch: 687/1000, loss: 1.2875, mfu: 14.84%\n",
      "epoch: 688/1000, loss: 1.2730, mfu: 13.37%\n",
      "epoch: 689/1000, loss: 1.2233, mfu: 14.36%\n",
      "epoch: 690/1000, loss: 1.2771, mfu: 14.59%\n",
      "epoch: 691/1000, loss: 1.2434, mfu: 14.48%\n",
      "epoch: 692/1000, loss: 1.1600, mfu: 15.38%\n",
      "epoch: 693/1000, loss: 1.2813, mfu: 15.28%\n",
      "epoch: 694/1000, loss: 1.3094, mfu: 14.36%\n",
      "epoch: 695/1000, loss: 1.1936, mfu: 15.92%\n",
      "epoch: 696/1000, loss: 1.2371, mfu: 15.14%\n",
      "epoch: 697/1000, loss: 1.2161, mfu: 13.91%\n",
      "epoch: 698/1000, loss: 1.2585, mfu: 14.17%\n",
      "epoch: 699/1000, loss: 1.1666, mfu: 14.91%\n",
      "epoch: 700/1000, loss: 1.2145, mfu: 11.84%\n",
      "epoch: 701/1000, loss: 1.2346, mfu: 12.16%\n",
      "epoch: 702/1000, loss: 1.2683, mfu: 6.97%\n",
      "epoch: 703/1000, loss: 1.2091, mfu: 13.57%\n",
      "epoch: 704/1000, loss: 1.1624, mfu: 15.07%\n",
      "epoch: 705/1000, loss: 1.2205, mfu: 15.57%\n",
      "epoch: 706/1000, loss: 1.2057, mfu: 14.97%\n",
      "epoch: 707/1000, loss: 1.2517, mfu: 16.27%\n",
      "epoch: 708/1000, loss: 1.2977, mfu: 15.89%\n",
      "epoch: 709/1000, loss: 1.2615, mfu: 14.37%\n",
      "epoch: 710/1000, loss: 1.2597, mfu: 16.25%\n",
      "epoch: 711/1000, loss: 1.2605, mfu: 14.94%\n",
      "epoch: 712/1000, loss: 1.2486, mfu: 11.76%\n",
      "epoch: 713/1000, loss: 1.2564, mfu: 15.80%\n",
      "epoch: 714/1000, loss: 1.1989, mfu: 14.87%\n",
      "epoch: 715/1000, loss: 1.2244, mfu: 14.84%\n",
      "epoch: 716/1000, loss: 1.2437, mfu: 11.69%\n",
      "epoch: 717/1000, loss: 1.2040, mfu: 13.71%\n",
      "epoch: 718/1000, loss: 1.2578, mfu: 15.09%\n",
      "epoch: 719/1000, loss: 1.1936, mfu: 16.36%\n",
      "epoch: 720/1000, loss: 1.2166, mfu: 13.84%\n",
      "epoch: 721/1000, loss: 1.1763, mfu: 15.84%\n",
      "epoch: 722/1000, loss: 1.1883, mfu: 14.83%\n",
      "epoch: 723/1000, loss: 1.2559, mfu: 15.35%\n",
      "epoch: 724/1000, loss: 1.2368, mfu: 14.64%\n",
      "epoch: 725/1000, loss: 1.2345, mfu: 15.04%\n",
      "epoch: 726/1000, loss: 1.1615, mfu: 15.07%\n",
      "epoch: 727/1000, loss: 1.2549, mfu: 14.73%\n",
      "epoch: 728/1000, loss: 1.2063, mfu: 14.21%\n",
      "epoch: 729/1000, loss: 1.2156, mfu: 14.80%\n",
      "epoch: 730/1000, loss: 1.1919, mfu: 15.07%\n",
      "epoch: 731/1000, loss: 1.2256, mfu: 16.13%\n",
      "epoch: 732/1000, loss: 1.2563, mfu: 15.83%\n",
      "epoch: 733/1000, loss: 1.2576, mfu: 15.17%\n",
      "epoch: 734/1000, loss: 1.2448, mfu: 15.46%\n",
      "epoch: 735/1000, loss: 1.1591, mfu: 16.43%\n",
      "epoch: 736/1000, loss: 1.2553, mfu: 15.86%\n",
      "epoch: 737/1000, loss: 1.2857, mfu: 15.40%\n",
      "epoch: 738/1000, loss: 1.2705, mfu: 14.21%\n",
      "epoch: 739/1000, loss: 1.2221, mfu: 16.86%\n",
      "epoch: 740/1000, loss: 1.1617, mfu: 14.02%\n",
      "epoch: 741/1000, loss: 1.1316, mfu: 11.31%\n",
      "epoch: 742/1000, loss: 1.1969, mfu: 16.24%\n",
      "epoch: 743/1000, loss: 1.2215, mfu: 17.27%\n",
      "epoch: 744/1000, loss: 1.1988, mfu: 15.04%\n",
      "epoch: 745/1000, loss: 1.1854, mfu: 14.21%\n",
      "epoch: 746/1000, loss: 1.2499, mfu: 14.86%\n",
      "epoch: 747/1000, loss: 1.2284, mfu: 15.12%\n",
      "epoch: 748/1000, loss: 1.2031, mfu: 13.72%\n",
      "epoch: 749/1000, loss: 1.2221, mfu: 14.98%\n",
      "epoch: 750/1000, loss: 1.2704, mfu: 13.88%\n",
      "epoch: 751/1000, loss: 1.2522, mfu: 16.65%\n",
      "epoch: 752/1000, loss: 1.2204, mfu: 15.63%\n",
      "epoch: 753/1000, loss: 1.2157, mfu: 14.15%\n",
      "epoch: 754/1000, loss: 1.1992, mfu: 13.45%\n",
      "epoch: 755/1000, loss: 1.2058, mfu: 15.50%\n",
      "epoch: 756/1000, loss: 1.1597, mfu: 15.51%\n",
      "epoch: 757/1000, loss: 1.1535, mfu: 15.64%\n",
      "epoch: 758/1000, loss: 1.2413, mfu: 17.19%\n",
      "epoch: 759/1000, loss: 1.2478, mfu: 14.28%\n",
      "epoch: 760/1000, loss: 1.1757, mfu: 14.80%\n",
      "epoch: 761/1000, loss: 1.1974, mfu: 14.35%\n",
      "epoch: 762/1000, loss: 1.2344, mfu: 16.50%\n",
      "epoch: 763/1000, loss: 1.2068, mfu: 15.73%\n",
      "epoch: 764/1000, loss: 1.1914, mfu: 14.14%\n",
      "epoch: 765/1000, loss: 1.2470, mfu: 15.33%\n",
      "epoch: 766/1000, loss: 1.2252, mfu: 15.47%\n",
      "epoch: 767/1000, loss: 1.2221, mfu: 15.26%\n",
      "epoch: 768/1000, loss: 1.1945, mfu: 16.16%\n",
      "epoch: 769/1000, loss: 1.1980, mfu: 16.30%\n",
      "epoch: 770/1000, loss: 1.1460, mfu: 15.74%\n",
      "epoch: 771/1000, loss: 1.1801, mfu: 15.10%\n",
      "epoch: 772/1000, loss: 1.2297, mfu: 14.69%\n",
      "epoch: 773/1000, loss: 1.2310, mfu: 15.17%\n",
      "epoch: 774/1000, loss: 1.2053, mfu: 15.33%\n",
      "epoch: 775/1000, loss: 1.1349, mfu: 16.09%\n",
      "epoch: 776/1000, loss: 1.2298, mfu: 14.15%\n",
      "epoch: 777/1000, loss: 1.2220, mfu: 14.66%\n",
      "epoch: 778/1000, loss: 1.1618, mfu: 14.80%\n",
      "epoch: 779/1000, loss: 1.1262, mfu: 15.39%\n",
      "epoch: 780/1000, loss: 1.2100, mfu: 16.28%\n",
      "epoch: 781/1000, loss: 1.2424, mfu: 15.36%\n",
      "epoch: 782/1000, loss: 1.2546, mfu: 12.76%\n",
      "epoch: 783/1000, loss: 1.2094, mfu: 15.08%\n",
      "epoch: 784/1000, loss: 1.2064, mfu: 14.76%\n",
      "epoch: 785/1000, loss: 1.2348, mfu: 16.52%\n",
      "epoch: 786/1000, loss: 1.1584, mfu: 13.96%\n",
      "epoch: 787/1000, loss: 1.2361, mfu: 14.29%\n",
      "epoch: 788/1000, loss: 1.1626, mfu: 15.36%\n",
      "epoch: 789/1000, loss: 1.2282, mfu: 14.83%\n",
      "epoch: 790/1000, loss: 1.2366, mfu: 14.42%\n",
      "epoch: 791/1000, loss: 1.1998, mfu: 15.80%\n",
      "epoch: 792/1000, loss: 1.2203, mfu: 15.34%\n",
      "epoch: 793/1000, loss: 1.2351, mfu: 16.08%\n",
      "epoch: 794/1000, loss: 1.2207, mfu: 16.13%\n",
      "epoch: 795/1000, loss: 1.1611, mfu: 14.85%\n",
      "epoch: 796/1000, loss: 1.1978, mfu: 16.08%\n",
      "epoch: 797/1000, loss: 1.1497, mfu: 15.17%\n",
      "epoch: 798/1000, loss: 1.2787, mfu: 14.67%\n",
      "epoch: 799/1000, loss: 1.1666, mfu: 15.41%\n",
      "epoch: 800/1000, loss: 1.2016, mfu: 15.80%\n",
      "epoch: 801/1000, loss: 1.2239, mfu: 15.30%\n",
      "epoch: 802/1000, loss: 1.1915, mfu: 14.05%\n",
      "epoch: 803/1000, loss: 1.1327, mfu: 15.34%\n",
      "epoch: 804/1000, loss: 1.2129, mfu: 15.19%\n",
      "epoch: 805/1000, loss: 1.1880, mfu: 15.24%\n",
      "epoch: 806/1000, loss: 1.2116, mfu: 14.16%\n",
      "epoch: 807/1000, loss: 1.2582, mfu: 15.10%\n",
      "epoch: 808/1000, loss: 1.2222, mfu: 15.39%\n",
      "epoch: 809/1000, loss: 1.1703, mfu: 15.86%\n",
      "epoch: 810/1000, loss: 1.1071, mfu: 14.02%\n",
      "epoch: 811/1000, loss: 1.2237, mfu: 16.01%\n",
      "epoch: 812/1000, loss: 1.2712, mfu: 15.32%\n",
      "epoch: 813/1000, loss: 1.1684, mfu: 15.90%\n",
      "epoch: 814/1000, loss: 1.1396, mfu: 15.23%\n",
      "epoch: 815/1000, loss: 1.2347, mfu: 14.78%\n",
      "epoch: 816/1000, loss: 1.2058, mfu: 14.52%\n",
      "epoch: 817/1000, loss: 1.1528, mfu: 16.19%\n",
      "epoch: 818/1000, loss: 1.2258, mfu: 14.92%\n",
      "epoch: 819/1000, loss: 1.1765, mfu: 15.52%\n",
      "epoch: 820/1000, loss: 1.2058, mfu: 13.66%\n",
      "epoch: 821/1000, loss: 1.1628, mfu: 15.18%\n",
      "epoch: 822/1000, loss: 1.1933, mfu: 16.00%\n",
      "epoch: 823/1000, loss: 1.2382, mfu: 16.67%\n",
      "epoch: 824/1000, loss: 1.2124, mfu: 14.74%\n",
      "epoch: 825/1000, loss: 1.2398, mfu: 16.03%\n",
      "epoch: 826/1000, loss: 1.2013, mfu: 16.21%\n",
      "epoch: 827/1000, loss: 1.2198, mfu: 15.57%\n",
      "epoch: 828/1000, loss: 1.2265, mfu: 16.04%\n",
      "epoch: 829/1000, loss: 1.2130, mfu: 15.26%\n",
      "epoch: 830/1000, loss: 1.1810, mfu: 15.66%\n",
      "epoch: 831/1000, loss: 1.2039, mfu: 16.47%\n",
      "epoch: 832/1000, loss: 1.2029, mfu: 14.50%\n",
      "epoch: 833/1000, loss: 1.1837, mfu: 13.34%\n",
      "epoch: 834/1000, loss: 1.1521, mfu: 15.55%\n",
      "epoch: 835/1000, loss: 1.1817, mfu: 13.96%\n",
      "epoch: 836/1000, loss: 1.1077, mfu: 16.80%\n",
      "epoch: 837/1000, loss: 1.1671, mfu: 15.54%\n",
      "epoch: 838/1000, loss: 1.1174, mfu: 14.50%\n",
      "epoch: 839/1000, loss: 1.2029, mfu: 15.77%\n",
      "epoch: 840/1000, loss: 1.2101, mfu: 14.80%\n",
      "epoch: 841/1000, loss: 1.1894, mfu: 15.30%\n",
      "epoch: 842/1000, loss: 1.1672, mfu: 15.58%\n",
      "epoch: 843/1000, loss: 1.1880, mfu: 14.76%\n",
      "epoch: 844/1000, loss: 1.1842, mfu: 14.63%\n",
      "epoch: 845/1000, loss: 1.1875, mfu: 14.14%\n",
      "epoch: 846/1000, loss: 1.1573, mfu: 14.04%\n",
      "epoch: 847/1000, loss: 1.1229, mfu: 14.99%\n",
      "epoch: 848/1000, loss: 1.1489, mfu: 15.14%\n",
      "epoch: 849/1000, loss: 1.1685, mfu: 13.81%\n",
      "epoch: 850/1000, loss: 1.1682, mfu: 14.30%\n",
      "epoch: 851/1000, loss: 1.1932, mfu: 14.21%\n",
      "epoch: 852/1000, loss: 1.2032, mfu: 16.20%\n",
      "epoch: 853/1000, loss: 1.1852, mfu: 15.17%\n",
      "epoch: 854/1000, loss: 1.1624, mfu: 14.88%\n",
      "epoch: 855/1000, loss: 1.1849, mfu: 13.82%\n",
      "epoch: 856/1000, loss: 1.1414, mfu: 15.84%\n",
      "epoch: 857/1000, loss: 1.1437, mfu: 15.44%\n",
      "epoch: 858/1000, loss: 1.1771, mfu: 16.20%\n",
      "epoch: 859/1000, loss: 1.1977, mfu: 16.79%\n",
      "epoch: 860/1000, loss: 1.1560, mfu: 14.16%\n",
      "epoch: 861/1000, loss: 1.1110, mfu: 14.44%\n",
      "epoch: 862/1000, loss: 1.1808, mfu: 13.21%\n",
      "epoch: 863/1000, loss: 1.1904, mfu: 14.91%\n",
      "epoch: 864/1000, loss: 1.2542, mfu: 14.26%\n",
      "epoch: 865/1000, loss: 1.2300, mfu: 12.01%\n",
      "epoch: 866/1000, loss: 1.1980, mfu: 14.56%\n",
      "epoch: 867/1000, loss: 1.1778, mfu: 16.34%\n",
      "epoch: 868/1000, loss: 1.2019, mfu: 15.24%\n",
      "epoch: 869/1000, loss: 1.2064, mfu: 16.41%\n",
      "epoch: 870/1000, loss: 1.2500, mfu: 14.46%\n",
      "epoch: 871/1000, loss: 1.1786, mfu: 10.97%\n",
      "epoch: 872/1000, loss: 1.1293, mfu: 14.42%\n",
      "epoch: 873/1000, loss: 1.1946, mfu: 14.17%\n",
      "epoch: 874/1000, loss: 1.1944, mfu: 15.70%\n",
      "epoch: 875/1000, loss: 1.1775, mfu: 14.16%\n",
      "epoch: 876/1000, loss: 1.2063, mfu: 15.65%\n",
      "epoch: 877/1000, loss: 1.1783, mfu: 15.74%\n",
      "epoch: 878/1000, loss: 1.1319, mfu: 14.99%\n",
      "epoch: 879/1000, loss: 1.1435, mfu: 15.83%\n",
      "epoch: 880/1000, loss: 1.1856, mfu: 14.37%\n",
      "epoch: 881/1000, loss: 1.1962, mfu: 15.87%\n",
      "epoch: 882/1000, loss: 1.1853, mfu: 14.64%\n",
      "epoch: 883/1000, loss: 1.2223, mfu: 15.60%\n",
      "epoch: 884/1000, loss: 1.1564, mfu: 15.61%\n",
      "epoch: 885/1000, loss: 1.1241, mfu: 16.16%\n",
      "epoch: 886/1000, loss: 1.1726, mfu: 14.92%\n",
      "epoch: 887/1000, loss: 1.1911, mfu: 15.20%\n",
      "epoch: 888/1000, loss: 1.0762, mfu: 14.31%\n",
      "epoch: 889/1000, loss: 1.1201, mfu: 16.08%\n",
      "epoch: 890/1000, loss: 1.1785, mfu: 14.97%\n",
      "epoch: 891/1000, loss: 1.1736, mfu: 16.02%\n",
      "epoch: 892/1000, loss: 1.1810, mfu: 15.20%\n",
      "epoch: 893/1000, loss: 1.1330, mfu: 14.53%\n",
      "epoch: 894/1000, loss: 1.1838, mfu: 11.57%\n",
      "epoch: 895/1000, loss: 1.1645, mfu: 13.28%\n",
      "epoch: 896/1000, loss: 1.1868, mfu: 14.98%\n",
      "epoch: 897/1000, loss: 1.1636, mfu: 14.94%\n",
      "epoch: 898/1000, loss: 1.1873, mfu: 13.67%\n",
      "epoch: 899/1000, loss: 1.2035, mfu: 16.46%\n",
      "epoch: 900/1000, loss: 1.1602, mfu: 15.14%\n",
      "epoch: 901/1000, loss: 1.1865, mfu: 15.48%\n",
      "epoch: 902/1000, loss: 1.1862, mfu: 15.56%\n",
      "epoch: 903/1000, loss: 1.1564, mfu: 15.64%\n",
      "epoch: 904/1000, loss: 1.2226, mfu: 15.17%\n",
      "epoch: 905/1000, loss: 1.2041, mfu: 15.82%\n",
      "epoch: 906/1000, loss: 1.1984, mfu: 13.79%\n",
      "epoch: 907/1000, loss: 1.1718, mfu: 13.58%\n",
      "epoch: 908/1000, loss: 1.1763, mfu: 14.57%\n",
      "epoch: 909/1000, loss: 1.1824, mfu: 13.63%\n",
      "epoch: 910/1000, loss: 1.1789, mfu: 13.84%\n",
      "epoch: 911/1000, loss: 1.1725, mfu: 15.45%\n",
      "epoch: 912/1000, loss: 1.1732, mfu: 14.13%\n",
      "epoch: 913/1000, loss: 1.1995, mfu: 13.89%\n",
      "epoch: 914/1000, loss: 1.1678, mfu: 15.87%\n",
      "epoch: 915/1000, loss: 1.1345, mfu: 15.72%\n",
      "epoch: 916/1000, loss: 1.1614, mfu: 15.55%\n",
      "epoch: 917/1000, loss: 1.1621, mfu: 15.47%\n",
      "epoch: 918/1000, loss: 1.2001, mfu: 14.91%\n",
      "epoch: 919/1000, loss: 1.1449, mfu: 14.89%\n",
      "epoch: 920/1000, loss: 1.1697, mfu: 15.33%\n",
      "epoch: 921/1000, loss: 1.1237, mfu: 16.11%\n",
      "epoch: 922/1000, loss: 1.2297, mfu: 13.36%\n",
      "epoch: 923/1000, loss: 1.1881, mfu: 14.06%\n",
      "epoch: 924/1000, loss: 1.1484, mfu: 14.34%\n",
      "epoch: 925/1000, loss: 1.1705, mfu: 14.16%\n",
      "epoch: 926/1000, loss: 1.1263, mfu: 15.37%\n",
      "epoch: 927/1000, loss: 1.2006, mfu: 15.79%\n",
      "epoch: 928/1000, loss: 1.1197, mfu: 14.09%\n",
      "epoch: 929/1000, loss: 1.2158, mfu: 14.52%\n",
      "epoch: 930/1000, loss: 1.2303, mfu: 15.40%\n",
      "epoch: 931/1000, loss: 1.1611, mfu: 16.25%\n",
      "epoch: 932/1000, loss: 1.1743, mfu: 14.85%\n",
      "epoch: 933/1000, loss: 1.1728, mfu: 15.04%\n",
      "epoch: 934/1000, loss: 1.1896, mfu: 15.40%\n",
      "epoch: 935/1000, loss: 1.1229, mfu: 15.99%\n",
      "epoch: 936/1000, loss: 1.1969, mfu: 13.69%\n",
      "epoch: 937/1000, loss: 1.1713, mfu: 14.73%\n",
      "epoch: 938/1000, loss: 1.1681, mfu: 14.58%\n",
      "epoch: 939/1000, loss: 1.1329, mfu: 15.88%\n",
      "epoch: 940/1000, loss: 1.1737, mfu: 15.61%\n",
      "epoch: 941/1000, loss: 1.2370, mfu: 16.07%\n",
      "epoch: 942/1000, loss: 1.1679, mfu: 15.58%\n",
      "epoch: 943/1000, loss: 1.1398, mfu: 14.64%\n",
      "epoch: 944/1000, loss: 1.1921, mfu: 14.74%\n",
      "epoch: 945/1000, loss: 1.1620, mfu: 14.93%\n",
      "epoch: 946/1000, loss: 1.1645, mfu: 14.28%\n",
      "epoch: 947/1000, loss: 1.1391, mfu: 16.25%\n",
      "epoch: 948/1000, loss: 1.1948, mfu: 14.94%\n",
      "epoch: 949/1000, loss: 1.1825, mfu: 17.52%\n",
      "epoch: 950/1000, loss: 1.1142, mfu: 16.61%\n",
      "epoch: 951/1000, loss: 1.1046, mfu: 15.29%\n",
      "epoch: 952/1000, loss: 1.1633, mfu: 16.29%\n",
      "epoch: 953/1000, loss: 1.1508, mfu: 12.32%\n",
      "epoch: 954/1000, loss: 1.1323, mfu: 8.97%\n",
      "epoch: 955/1000, loss: 1.1511, mfu: 14.17%\n",
      "epoch: 956/1000, loss: 1.2526, mfu: 11.56%\n",
      "epoch: 957/1000, loss: 1.1269, mfu: 14.66%\n",
      "epoch: 958/1000, loss: 1.1801, mfu: 13.14%\n",
      "epoch: 959/1000, loss: 1.1336, mfu: 14.92%\n",
      "epoch: 960/1000, loss: 1.1701, mfu: 15.26%\n",
      "epoch: 961/1000, loss: 1.1321, mfu: 13.68%\n",
      "epoch: 962/1000, loss: 1.1527, mfu: 13.73%\n",
      "epoch: 963/1000, loss: 1.1808, mfu: 13.33%\n",
      "epoch: 964/1000, loss: 1.1584, mfu: 11.58%\n",
      "epoch: 965/1000, loss: 1.1265, mfu: 14.12%\n",
      "epoch: 966/1000, loss: 1.0793, mfu: 11.07%\n",
      "epoch: 967/1000, loss: 1.1594, mfu: 13.01%\n",
      "epoch: 968/1000, loss: 1.2076, mfu: 13.62%\n",
      "epoch: 969/1000, loss: 1.1401, mfu: 13.88%\n",
      "epoch: 970/1000, loss: 1.1645, mfu: 14.27%\n",
      "epoch: 971/1000, loss: 1.1244, mfu: 14.07%\n",
      "epoch: 972/1000, loss: 1.1688, mfu: 12.18%\n",
      "epoch: 973/1000, loss: 1.2166, mfu: 13.09%\n",
      "epoch: 974/1000, loss: 1.1649, mfu: 12.75%\n",
      "epoch: 975/1000, loss: 1.1907, mfu: 10.89%\n",
      "epoch: 976/1000, loss: 1.1916, mfu: 12.68%\n",
      "epoch: 977/1000, loss: 1.2240, mfu: 13.25%\n",
      "epoch: 978/1000, loss: 1.1062, mfu: 12.75%\n",
      "epoch: 979/1000, loss: 1.1116, mfu: 13.70%\n",
      "epoch: 980/1000, loss: 1.1546, mfu: 13.67%\n",
      "epoch: 981/1000, loss: 1.1177, mfu: 12.93%\n",
      "epoch: 982/1000, loss: 1.1143, mfu: 13.88%\n",
      "epoch: 983/1000, loss: 1.1673, mfu: 12.91%\n",
      "epoch: 984/1000, loss: 1.1546, mfu: 13.63%\n",
      "epoch: 985/1000, loss: 1.1258, mfu: 12.79%\n",
      "epoch: 986/1000, loss: 1.1363, mfu: 13.13%\n",
      "epoch: 987/1000, loss: 1.1492, mfu: 13.38%\n",
      "epoch: 988/1000, loss: 1.2192, mfu: 12.76%\n",
      "epoch: 989/1000, loss: 1.1702, mfu: 14.38%\n",
      "epoch: 990/1000, loss: 1.2038, mfu: 12.15%\n",
      "epoch: 991/1000, loss: 1.1525, mfu: 12.96%\n",
      "epoch: 992/1000, loss: 1.2037, mfu: 13.20%\n",
      "epoch: 993/1000, loss: 1.1476, mfu: 14.01%\n",
      "epoch: 994/1000, loss: 1.2034, mfu: 13.56%\n",
      "epoch: 995/1000, loss: 1.1571, mfu: 14.36%\n",
      "epoch: 996/1000, loss: 1.2180, mfu: 12.31%\n",
      "epoch: 997/1000, loss: 1.1666, mfu: 12.81%\n",
      "epoch: 998/1000, loss: 1.1243, mfu: 13.71%\n",
      "epoch: 999/1000, loss: 1.1815, mfu: 13.85%\n",
      "epoch: 1000/1000, loss: 1.1742, mfu: 13.11%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHWCAYAAABACtmGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB7g0lEQVR4nO3dd1zU9R8H8NeXdcw79gZBQEARxI3b3Fo5mmY5mqaWtrNdVlj9WjbUlrbM0hxljtwTNzhQcaCA7HnHHnff3x8HXzg2Bhzj9Xw8eMR9v9/73vvoSl5+Pp/3RxBFUQQRERERERHVyUDfBRAREREREbV1DE5EREREREQNYHAiIiIiIiJqAIMTERERERFRAxiciIiIiIiIGsDgRERERERE1AAGJyIiIiIiogYwOBERERERETWAwYmIiIiIiKgBDE5ERNRmzJ49G15eXrf03LfeeguCIDRvQUREROUYnIiIqEGCIDTqa9++ffouVS9mz54NS0tLfZdBREQtSBBFUdR3EURE1Lb98ssvOo9/+ukn7Ny5Ez///LPO8TFjxsDJyemWX6e0tBQajQYymazJzy0rK0NZWRlMTU1v+fVv1ezZs7F+/Xrk5eW1+msTEVHrMNJ3AURE1PY9+OCDOo+PHj2KnTt31jheXUFBAczNzRv9OsbGxrdUHwAYGRnByIh/rBERUcvgVD0iImoWI0aMQFBQEE6dOoVhw4bB3Nwcr7zyCgBg8+bNmDRpElxdXSGTyeDj44MlS5ZArVbr3KP6GqcbN25AEAT873//wzfffAMfHx/IZDL069cPJ06c0HlubWucBEHAggULsGnTJgQFBUEmk6FHjx7Yvn17jfr37duHvn37wtTUFD4+Pli5cmWzr5tat24d+vTpAzMzM9jb2+PBBx9EYmKizjUpKSmYM2cO3N3dIZPJ4OLigsmTJ+PGjRvSNSdPnsS4ceNgb28PMzMzeHt74+GHH262OomIqCb+1RwRETWbzMxMTJgwAffffz8efPBBadre6tWrYWlpiWeffRaWlpbYs2cP3njjDahUKnz00UcN3nfNmjXIzc3FE088AUEQ8OGHH2LatGmIjY1tcJTq0KFD2LBhA+bNmwcrKyssW7YMd911F+Lj42FnZwcAiIyMxPjx4+Hi4oK3334barUa77zzDhwcHP77D6Xc6tWrMWfOHPTr1w/h4eFITU3F559/jsOHDyMyMhLW1tYAgLvuugvR0dF46qmn4OXlhbS0NOzcuRPx8fHS47Fjx8LBwQEvv/wyrK2tcePGDWzYsKHZaiUiolqIRERETTR//nyx+h8hw4cPFwGIK1asqHF9QUFBjWNPPPGEaG5uLhYVFUnHZs2aJXbp0kV6fP36dRGAaGdnJ2ZlZUnHN2/eLAIQ//77b+nYm2++WaMmAKKJiYl49epV6diZM2dEAOIXX3whHbvjjjtEc3NzMTExUTp25coV0cjIqMY9azNr1izRwsKizvMlJSWio6OjGBQUJBYWFkrHt2zZIgIQ33jjDVEURTE7O1sEIH700Ud13mvjxo0iAPHEiRMN1kVERM2HU/WIiKjZyGQyzJkzp8ZxMzMz6fvc3FxkZGRg6NChKCgowKVLlxq873333QcbGxvp8dChQwEAsbGxDT539OjR8PHxkR4HBwdDLpdLz1Wr1di1axemTJkCV1dX6TpfX19MmDChwfs3xsmTJ5GWloZ58+bpNK+YNGkSAgIC8M8//wDQ/pxMTEywb98+ZGdn13qvipGpLVu2oLS0tFnqIyKihjE4ERFRs3Fzc4OJiUmN49HR0Zg6dSoUCgXkcjkcHBykxhJKpbLB+3p6euo8rghRdYWL+p5b8fyK56alpaGwsBC+vr41rqvt2K2Ii4sDAPj7+9c4FxAQIJ2XyWT44IMPsG3bNjg5OWHYsGH48MMPkZKSIl0/fPhw3HXXXXj77bdhb2+PyZMnY9WqVSguLm6WWomIqHYMTkRE1GyqjixVyMnJwfDhw3HmzBm88847+Pvvv7Fz50588MEHAACNRtPgfQ0NDWs9LjZiR43/8lx9WLRoES5fvozw8HCYmpri9ddfR2BgICIjIwFoG16sX78eERERWLBgARITE/Hwww+jT58+bIdORNSCGJyIiKhF7du3D5mZmVi9ejUWLlyI22+/HaNHj9aZeqdPjo6OMDU1xdWrV2ucq+3YrejSpQsAICYmpsa5mJgY6XwFHx8fPPfcc/j3339x/vx5lJSU4OOPP9a5ZuDAgXjvvfdw8uRJ/Prrr4iOjsbatWubpV4iIqqJwYmIiFpUxYhP1RGekpISfP311/oqSYehoSFGjx6NTZs2ISkpSTp+9epVbNu2rVleo2/fvnB0dMSKFSt0ptRt27YNFy9exKRJkwBo970qKirSea6Pjw+srKyk52VnZ9cYLevVqxcAcLoeEVELYjtyIiJqUYMGDYKNjQ1mzZqFp59+GoIg4Oeff25TU+Xeeust/Pvvvxg8eDCefPJJqNVqfPnllwgKCkJUVFSj7lFaWop33323xnFbW1vMmzcPH3zwAebMmYPhw4dj+vTpUjtyLy8vPPPMMwCAy5cvY9SoUbj33nvRvXt3GBkZYePGjUhNTcX9998PAPjxxx/x9ddfY+rUqfDx8UFubi6+/fZbyOVyTJw4sdl+JkREpIvBiYiIWpSdnR22bNmC5557Dq+99hpsbGzw4IMPYtSoURg3bpy+ywMA9OnTB9u2bcPzzz+P119/HR4eHnjnnXdw8eLFRnX9A7SjaK+//nqN4z4+Ppg3bx5mz54Nc3NzLF26FC+99BIsLCwwdepUfPDBB1KnPA8PD0yfPh27d+/Gzz//DCMjIwQEBOCPP/7AXXfdBUDbHOL48eNYu3YtUlNToVAo0L9/f/z666/w9vZutp8JERHpEsS29Fd+REREbciUKVMQHR2NK1eu6LsUIiLSM65xIiIiAlBYWKjz+MqVK9i6dStGjBihn4KIiKhN4YgTERERABcXF8yePRtdu3ZFXFwcli9fjuLiYkRGRsLPz0/f5RERkZ5xjRMRERGA8ePH47fffkNKSgpkMhnCwsLw/vvvMzQREREAjjgRERERERE1iGuciIiIiIiIGsDgRERERERE1IBOt8ZJo9EgKSkJVlZWEARB3+UQEREREZGeiKKI3NxcuLq6wsCg/jGlTheckpKS4OHhoe8yiIiIiIiojUhISIC7u3u913S64GRlZQVA+8ORy+V6roaIiIiIiPRFpVLBw8NDygj16XTBqWJ6nlwuZ3AiIiIiIqJGLeFhcwgiIiIiIqIGMDgRERERERE1gMGJiIiIiIioAQxOREREREREDWBwIiIiIiIiagCDExERERERUQMYnIiIiIiIiBrA4ERERERERNQABiciIiIiIqIGMDgRERERERE1gMGJiIiIiIioAQxOREREREREDWBwIiIiIiIiaoCRvgvozH6KuIHNUUmQGRlgaqgb7uzlCpmRob7LIiIiIiKiahic9ChZWYRTcdkAgCPXMrFi/zV8O7MvujpY6rkyIiIiIiKqilP19OjOEFd89UBvPD+2G+wtZbiWno8Hvj2G9NxifZdGRERERERVMDjpUaCLHJOCXbDgNj9sXzQUPg4WSFEVYeHaSKg1or7LIyIiIiKicgxObYS9pQwrH+oDcxNDHLmWie8Oxuq7JCIiIiIiKsfg1Ib4OlrhrTt6AAC+2HMVmXmcskdERERE1BYwOLUxd/dxR083BfKKy/DjkRv6LoeIiIiIiMDg1OYYGAiYN8IHAPBjRBzyi8v0XBERERERETE4tUFjezjD294CysJS/HY8Xt/lEBERERF1em0mOC1duhSCIGDRokV1XrN69WoIgqDzZWpq2npFthJDAwGPD+sKAPj+0HWUlGn0XBERERERUefWJoLTiRMnsHLlSgQHBzd4rVwuR3JysvQVFxfXChW2vqmhbnCwkiFZWYR/L6TouxwiIiIiok5N78EpLy8PM2bMwLfffgsbG5sGrxcEAc7OztKXk5NTK1TZ+kyNDXFvX3cAwKbIRD1XQ0RERETUuek9OM2fPx+TJk3C6NGjG3V9Xl4eunTpAg8PD0yePBnR0dH1Xl9cXAyVSqXz1V5M6eUGANgXk46s/BI9V0NERERE1HnpNTitXbsWp0+fRnh4eKOu9/f3xw8//IDNmzfjl19+gUajwaBBg3Dz5s06nxMeHg6FQiF9eXh4NFf5Lc7PyQpBbnKUaUT8czZJ3+UQEREREXVaegtOCQkJWLhwIX799ddGN3gICwvDzJkz0atXLwwfPhwbNmyAg4MDVq5cWedzFi9eDKVSKX0lJCQ011toFRWjTlvPcZ0TEREREZG+6C04nTp1CmlpaejduzeMjIxgZGSE/fv3Y9myZTAyMoJarW7wHsbGxggNDcXVq1frvEYmk0Eul+t8tScjAxwBAKfislFY0vDPhIiIiIiImp/egtOoUaNw7tw5REVFSV99+/bFjBkzEBUVBUNDwwbvoVarce7cObi4uLRCxfrR1d4CLgpTlKg1OBmXpe9yiIiIiIg6JSN9vbCVlRWCgoJ0jllYWMDOzk46PnPmTLi5uUlroN555x0MHDgQvr6+yMnJwUcffYS4uDg8+uijrV5/axEEAYN87PHn6Zs4fDUTQ/0c9F0SEREREVGno7fg1Bjx8fEwMKgcFMvOzsZjjz2GlJQU2NjYoE+fPjhy5Ai6d++uxypb3mBfO/x5+iaOXMvQdylERERERJ2SIIqiqO8iWpNKpYJCoYBSqWw3651SVUUY8P5uCAIQ+foYWJub6LskIiIiIqJ2rynZQO/7OFHDnOSm8HGwgCgCR2Mz9V0OEREREVGnw+DUTgzxtQcAHL7K4ERERERE1NoYnNqJQRXBieuciIiIiIhaHYNTOzGwqx0MBCA2PR8pyiJ9l0NERERE1KkwOLUTCjNj9HRTAAAOX+WoExERERFRa2JwakcG+tgBAE7FZ+u5EiIiIiKizoXBqR3p4aodcbqYrNJzJUREREREnQuDUzvS3cUKABCTkguNplNtv0VEREREpFcMTu2Il50FZEYGKChRIy6rQN/lEBERERF1GgxO7YiRoQECXbQ7GkclcJ0TEREREVFrYXBqZ/p72wIAjl/P0nMlRERERESdB4NTO9PPSxucjjE4ERERERG1Ggandqa3pzUA7Ua4BSVl+i2GiIiIiKiTYHBqZ+wsZbCzMAEAXE3L03M1RERERESdA4NTO+TnZAkAuJLK4ERERERE1BoYnNohP0ftfk6X03L1XAkRERERUefA4NQOVYw4XUvL13MlRERERESdA4NTO+Rpaw4ASOAmuERERERErYLBqR3qYmcBAIjPKoAoinquhoiIiIio42NwaofcrM1gIACFpWqk5xbruxwiIiIiog6PwakdMjEygIvCDIB21ImIiIiIiFoWg1M71cVOu84pLpPBiYiIiIiopTE4tVMVDSI44kRERERE1PIYnNopTzsGJyIiIiKi1sLg1E51sa3srEdERERERC2LwamdqpiqxzVOREREREQtj8GpnaqYqpeRV4z84jI9V0NERERE1LExOLVTCjNjKMyMAQAJ2Rx1IiIiIiJqSQxO7RhbkhMRERERtQ4Gp3asYp1TAhtEEBERERG1KAandowNIoiIiIiIWgeDUzsmTdXjiBMRERERUYticGrHPMv3cuJUPSIiIiKilsXg1I5VtCS/mV0AtUbUczVERERERB0Xg1M75iw3hYmhAUrVIpKVhfouh4iIiIiow2JwascMDQS425gBAOLZIIKIiIiIqMUwOLVznmwQQURERETU4hic2jkPm8p1TkRERERE1DLaTHBaunQpBEHAokWL6r1u3bp1CAgIgKmpKXr27ImtW7e2ToFtlIu1KQAgWVmk50qIiIiIiDquNhGcTpw4gZUrVyI4OLje644cOYLp06fjkUceQWRkJKZMmYIpU6bg/PnzrVRp2+Oi0AanFAYnIiIiIqIWo/fglJeXhxkzZuDbb7+FjY1Nvdd+/vnnGD9+PF544QUEBgZiyZIl6N27N7788stWqrbtcVFom0NwxImIiIiIqOXoPTjNnz8fkyZNwujRoxu8NiIiosZ148aNQ0RERJ3PKS4uhkql0vnqSCpGnJKVhRBF7uVERERERNQSjPT54mvXrsXp06dx4sSJRl2fkpICJycnnWNOTk5ISUmp8znh4eF4++23/1OdbZmTXBuciko1UBaWwtrcRM8VERERERF1PHobcUpISMDChQvx66+/wtTUtMVeZ/HixVAqldJXQkJCi72WPpgaG8LOQhuWknI4XY+IiIiIqCXobcTp1KlTSEtLQ+/evaVjarUaBw4cwJdffoni4mIYGhrqPMfZ2Rmpqak6x1JTU+Hs7Fzn68hkMshksuYtvo1xVpgiM78EKapCdHeV67scIiIiIqIOR28jTqNGjcK5c+cQFRUlffXt2xczZsxAVFRUjdAEAGFhYdi9e7fOsZ07dyIsLKy1ym6TKhpEcMSJiIiIiKhl6G3EycrKCkFBQTrHLCwsYGdnJx2fOXMm3NzcEB4eDgBYuHAhhg8fjo8//hiTJk3C2rVrcfLkSXzzzTetXn9bwpbkREREREQtS+9d9eoTHx+P5ORk6fGgQYOwZs0afPPNNwgJCcH69euxadOmGgGss3EuD05JykI9V0JERERE1DHptatedfv27av3MQDcc889uOeee1qnoHbC1ZojTkRERERELalNjzhR4zjLuQkuEREREVFLYnDqACqm6qWqGJyIiIiIiFoCg1MH4GilbbdeUKJGXnGZnqshIiIiIup4GJw6AAuZESxl2uVqHHUiIiIiImp+DE4dRMWoU5qqWM+VEBERERF1PAxOHYSjvDw45XLEiYiIiIiouTE4dRCOVtoGERxxIiIiIiJqfgxOHYRT+YgT1zgRERERETU/BqcOQhpxyuWIExERERFRc2Nw6iAcOeJERERERNRiGJw6iIoRp3SOOBERERERNTsGpw6Ca5yIiIiIiFoOg1MH4SjXjjjll6iRV1ym52qIiIiIiDoWBqcOwlJmBAsTQwBAGkediIiIiIiaFYNTB1Ix6sTOekREREREzYvBqQNxtOI6JyIiIiKilsDg1IFUjDixsx4RERERUfNicOpAnDjiRERERETUIhicOpCKTXC5xomIiIiIqHkxOHUgTuVT9TjiRERERETUvBicOhAHK444ERERERG1BAanDqRixClNxeBERERERNScGJw6kIp25HnFZcgvLtNzNUREREREHQeDUwdiKTOCuYkhAE7XIyIiIiJqTgxOHYggCNKoUxobRBARERERNRsGpw6mYhPcVI44ERERERE1GwanDoYjTkREREREzY/BqYOROutxxImIiIiIqNkwOHUwHHEiIiIiImp+DE4djLNCO+KUrGRwIiIiIiJqLgxOHYyLwgwAgxMRERERUXNicOpgXK21I04pyiJoNKKeqyEiIiIi6hgYnDoYJ7kpBAEoUWuQkc8GEUREREREzYHBqYMxNjSAk1X5OqccTtcjIiIiImoODE4dUMV0vaScQj1XQkRERETUMTA4dUBsEEFERERE1LwYnDogh/K9nDLyuMaJiIiIiKg5MDh1QBXBKT2XwYmIiIiIqDkwOHVAUnDiiBMRERERUbPQa3Bavnw5goODIZfLIZfLERYWhm3bttV5/erVqyEIgs6XqalpK1bcPnDEiYiIiIioeRnp88Xd3d2xdOlS+Pn5QRRF/Pjjj5g8eTIiIyPRo0ePWp8jl8sRExMjPRYEobXKbTccLBmciIiIiIiak16D0x133KHz+L333sPy5ctx9OjROoOTIAhwdnZujfLaLcfyEafM/BKoNSIMDRguiYiIiIj+izazxkmtVmPt2rXIz89HWFhYndfl5eWhS5cu8PDwwOTJkxEdHV3vfYuLi6FSqXS+OjpbCxMIAqDWiMguKNF3OURERERE7Z7eg9O5c+dgaWkJmUyGuXPnYuPGjejevXut1/r7++OHH37A5s2b8csvv0Cj0WDQoEG4efNmnfcPDw+HQqGQvjw8PFrqrbQZRoYGsLMwAcDpekREREREzUEQRVHUZwElJSWIj4+HUqnE+vXr8d1332H//v11hqeqSktLERgYiOnTp2PJkiW1XlNcXIzi4srwoFKp4OHhAaVSCblc3mzvo60Z/9kBXErJxU8P98ewbg76LoeIiIiIqM1RqVRQKBSNygZ6XeMEACYmJvD19QUA9OnTBydOnMDnn3+OlStXNvhcY2NjhIaG4urVq3VeI5PJIJPJmq3e9sLBSoZLKbkccSIiIiIiagZ6n6pXnUaj0Rkhqo9arca5c+fg4uLSwlW1P9zLiYiIiIio+eh1xGnx4sWYMGECPD09kZubizVr1mDfvn3YsWMHAGDmzJlwc3NDeHg4AOCdd97BwIED4evri5ycHHz00UeIi4vDo48+qs+30SZxLyciIiIiouaj1+CUlpaGmTNnIjk5GQqFAsHBwdixYwfGjBkDAIiPj4eBQeWgWHZ2Nh577DGkpKTAxsYGffr0wZEjRxq1Hqqz4V5ORERERETNR+/NIVpbUxaAtWeboxKxcG0Uwrra4bfHB+q7HCIiIiKiNqcp2aDNrXGi5uFoZQoASFEV6bkSIiIiIqL2j8Gpg/KwNQMAJGYXQq3pVIOKRERERETNjsGpg3JRmMHIQECJWoNUjjoREREREf0nDE4dlKGBADcb7ahTfFaBnqshIiIiImrfGJw6ME9bcwBAAoMTEREREdF/wuDUgXkwOBERERERNQsGpw6sYsSJU/WIiIiIiP4bBqcOjMGJiIiIiKh5MDh1YB425VP1sgv1XAkRERERUfvG4NSBVYw4pecWo7BEredqiIiIiIjaLwanDkxhbgy5qREATtcjIiIiIvovGJw6OB9HSwDApRSVnishIiIiImq/GJw6uGA3BQDgTIJSz5UQEREREbVfDE4dXLC7NQDg7M0cvdZBRERERNSeMTh1cCEe2hGn6CQVNBpRz9UQEREREbVPDE4dXBc7CxgZCCgsVSNFVaTvcoiIiIiI2iUGpw7O2NAAnnbatuSx6fl6roaIiIiIqH1icOoEfBy0nfViM/L0XAkRERERUfvE4NQJdHWwAABcS2NwIiIiIiK6FQxOnUA3RysAwMWUXD1XQkRERETUPjE4dQLdXeUAgIvsrEdEREREdEsYnDoBX0dLmBgZILe4DAnZBfouh4iIiIio3WFw6gSMDQ3g76SdrnchSaXnaoiIiIiI2h8Gp07Cz7Gisx5bkhMRERERNRWDUyfhba/trHedwYmIiIiIqMkYnDoJbwcGJyIiIiKiW8Xg1El42TE4ERERERHdKganTqJiql5WfglyCkr0XA0RERERUfvC4NRJWMiM4CSXAeCoExERERFRUzE4dSIVo043MhmciIiIiIiagsGpE5E666UzOBERERERNQWDUydSEZy4lxMRERERUdMwOHUi3ZysAACn4rIhiqKeqyEiIiIiaj8YnDqRgV3tYG5iiGRlEc7eVOq7HCIiIiKidoPBqRMxNTbE8G4OAICDV9L1XA0RERERUfvB4NTJBLkpAADX2CCCiIiIiKjRGJw6GR+H8gYR6Xl6roSIiIiIqP1gcOpkfBwsAWhHnNgggoiIiIiocfQanJYvX47g4GDI5XLI5XKEhYVh27Zt9T5n3bp1CAgIgKmpKXr27ImtW7e2UrUdg6edOQwNBOQVlyFVVazvcoiIiIiI2gW9Bid3d3csXboUp06dwsmTJ3Hbbbdh8uTJiI6OrvX6I0eOYPr06XjkkUcQGRmJKVOmYMqUKTh//nwrV95+yYwMpel6UQk5+i2GiIiIiKidEMQ2Nl/L1tYWH330ER555JEa5+677z7k5+djy5Yt0rGBAweiV69eWLFiRaPur1KpoFAooFQqIZfLm63u9uS1Tefwy9F4PDzYG2/c0V3f5RARERER6UVTskGbWeOkVquxdu1a5OfnIywsrNZrIiIiMHr0aJ1j48aNQ0RERJ33LS4uhkql0vnq7Pp72wEAjt/I1HMlRERERETtg96D07lz52BpaQmZTIa5c+di48aN6N699lGQlJQUODk56RxzcnJCSkpKnfcPDw+HQqGQvjw8PJq1/vYoxF3bkvxyah7UmjY14EhERERE1CbpPTj5+/sjKioKx44dw5NPPolZs2bhwoULzXb/xYsXQ6lUSl8JCQnNdu/2yt3GHCaGBigp0yApp1Df5RARERERtXlG+i7AxMQEvr6+AIA+ffrgxIkT+Pzzz7Fy5coa1zo7OyM1NVXnWGpqKpydneu8v0wmg0wma96i2zlDAwFe9ua4nJqHa+l58LA113dJRERERERtmt5HnKrTaDQoLq69TXZYWBh2796tc2znzp11romiunW1r9zPiYiIiIiI6ndLI04JCQkQBAHu7u4AgOPHj2PNmjXo3r07Hn/88UbfZ/HixZgwYQI8PT2Rm5uLNWvWYN++fdixYwcAYObMmXBzc0N4eDgAYOHChRg+fDg+/vhjTJo0CWvXrsXJkyfxzTff3Mrb6NR8HC2AaCA2PU/fpRARERERtXm3NOL0wAMPYO/evQC0DRvGjBmD48eP49VXX8U777zT6PukpaVh5syZ8Pf3x6hRo3DixAns2LEDY8aMAQDEx8cjOTlZun7QoEFYs2YNvvnmG4SEhGD9+vXYtGkTgoKCbuVtdGoVI06xHHEiIiIiImrQLe3jZGNjg6NHj8Lf3x/Lli3D77//jsOHD+Pff//F3LlzERsb2xK1Ngvu46QVGZ+NqV8fgaOVDMdfHd3wE4iIiIiIOpgW38eptLRUariwa9cu3HnnnQCAgIAAnREiaru6OmhHnNJyi5FbVKrnaoiIiIiI2rZbCk49evTAihUrcPDgQezcuRPjx48HACQlJcHOzq5ZC6SWoTAzhr2lNvyyQQQRERERUf1uKTh98MEHWLlyJUaMGIHp06cjJCQEAPDXX3+hf//+zVogtZzg8o1w91xK03MlRERERERt2y111RsxYgQyMjKgUqlgY2MjHX/88cdhbs49gdqLyb1csedSGjZG3sQzo/0gCIK+SyIiIiIiapNuacSpsLAQxcXFUmiKi4vDZ599hpiYGDg6OjZrgdRyRgc6wUAAErIKkZ5b+95ZRERERER0i8Fp8uTJ+OmnnwAAOTk5GDBgAD7++GNMmTIFy5cvb9YCqeVYyIykJhHRySo9V0NERERE1HbdUnA6ffo0hg4dCgBYv349nJycEBcXh59++gnLli1r1gKpZXV30bZdvJDE4EREREREVJdbCk4FBQWwsrICAPz777+YNm0aDAwMMHDgQMTFxTVrgdSyergyOBERERERNeSWgpOvry82bdqEhIQE7NixA2PHjgUApKWldepNZduj7uXBaefFVFzPYFtyIiIiIqLa3FJweuONN/D888/Dy8sL/fv3R1hYGADt6FNoaGizFkgtq2KqXkmZBiP/tw/KQm6GS0RERERU3S21I7/77rsxZMgQJCcnS3s4AcCoUaMwderUZiuOWp5d+Sa4Fa6m5aJPF1s9VUNERERE1Dbd0ogTADg7OyM0NBRJSUm4efMmAKB///4ICAhotuKodYR1tZO+j8ss0GMlRERERERt0y0FJ41Gg3feeQcKhQJdunRBly5dYG1tjSVLlkCj0TR3jdTCPr43BBV73zI4ERERERHVdEtT9V599VV8//33WLp0KQYPHgwAOHToEN566y0UFRXhvffea9YiqWW5WpvhhXH++HB7DOIy2SCCiIiIiKi6WwpOP/74I7777jvceeed0rHg4GC4ublh3rx5DE7tkJedBQCwsx4RERERUS1uaapeVlZWrWuZAgICkJWV9Z+LotbX000BAIhOUiG/uEzP1RARERERtS23FJxCQkLw5Zdf1jj+5ZdfIjg4+D8XRa3Pw9Yc7jZmKNOI2ByVpO9yiIiIiIjalFuaqvfhhx9i0qRJ2LVrl7SHU0REBBISErB169ZmLZBaz1A/e/x2PAGvbDyHoX728LA113dJRERERERtwi2NOA0fPhyXL1/G1KlTkZOTg5ycHEybNg3R0dH4+eefm7tGaiUvjAuQuutFJuTotRYiIiIiorZEEEVRbK6bnTlzBr1794ZarW6uWzY7lUoFhUIBpVIJuVyu73LanMUbzuK34wl46jZfPDfWX9/lEBERERG1mKZkg1veAJc6pm5OVgCAmJRcPVdCRERERNR2MDiRjorgdCUtT8+VEBERERG1HQxOpKMiON3IzEdBSRnK1Bo9V0REREREpH9N6qo3bdq0es/n5OT8l1qoDbC3NIGNuTGyC0rR/Y0d8HeywtaFQ2FoIOi7NCIiIiIivWlScFIoFA2enzlz5n8qiPRLEAT4OlrixI1sAEBMai4SswvhacfW5ERERETUeTUpOK1ataql6qA2RK3RbbR4PTOfwYmIiIiIOjWucaIa5o3w1Xl8PZ2NIoiIiIioc2NwohpGd3fC8VdG4fFhXQEANzIL9FwREREREZF+MThRrRzlpuhqbwEAuMrW5ERERETUyTE4UZ2C3a0BAKfjs1FSxrbkRERERNR5MThRnQKcrWBrYYKCEjW6vbYNuUWl+i6JiIiIiEgvGJyoTgYGAkZ0c5Ae93zrX1xMVumxIiIiIiIi/WBwonq9fnt3DPC2lR5/tuuyHqshIiIiItIPBieql42FCdY8NhC3B7sAAA5eyUBRqVrPVRERERERtS4GJ2qQoYGAL6aHwtFKhoISNU7HZ+u7JCIiIiKiVsXgRI0iCAJCPa0BAJeSc/VbDBERERFRK2Nwokbzd7ICAFxOZXAiIiIios7FSN8FUPvhVx6c1p5IgMzIAC+OD4CFjB8hIiIiIur4OOJEjebvbCV9/2NEHEZ9vJ+NIoiIiIioU9BrcAoPD0e/fv1gZWUFR0dHTJkyBTExMfU+Z/Xq1RAEQefL1NS0lSru3PwcLfHgQE+4KLQ/7xRVEaIScvRbFBERERFRK9BrcNq/fz/mz5+Po0ePYufOnSgtLcXYsWORn59f7/PkcjmSk5Olr7i4uFaquHMTBAHvTumJiMWjMLGnMwCwwx4RERERdQp6XaCyfft2ncerV6+Go6MjTp06hWHDhtX5PEEQ4Ozs3KjXKC4uRnFxsfRYpVLdWrGko7enDbaeS8HpOAYnIiIiIur42tQaJ6VSCQCwtbWt97q8vDx06dIFHh4emDx5MqKjo+u8Njw8HAqFQvry8PBo1po7q95dbAAAp+NzIIqinqshIiIiImpZgthGfuvVaDS48847kZOTg0OHDtV5XUREBK5cuYLg4GAolUr873//w4EDBxAdHQ13d/ca19c24uTh4QGlUgm5XN4i76UzKC5To+eb/6JErYGbtRk+va8X+nvXH3iJiIiIiNoSlUoFhULRqGzQZoLTk08+iW3btuHQoUO1BqC6lJaWIjAwENOnT8eSJUsavL4pPxyqX58lO5GZXyI9vqePO96f1hPGhm1qIJOIiIiIqFZNyQZt4jfcBQsWYMuWLdi7d2+TQhMAGBsbIzQ0FFevXm2h6qgucwZ7Sd8LArDu1E1sjEzUX0FERERERC1Er8FJFEUsWLAAGzduxJ49e+Dt7d3ke6jVapw7dw4uLi4tUCHV54nhPti+aCiuh0/EUyN9AQC7LqTquSoiIiIiouan1+A0f/58/PLLL1izZg2srKyQkpKClJQUFBYWStfMnDkTixcvlh6/8847+PfffxEbG4vTp0/jwQcfRFxcHB599FF9vIVOzdjQAAHOcgiCgLE9tF0OD17J4Ka4RERERNTh6DU4LV++HEqlEiNGjICLi4v09fvvv0vXxMfHIzk5WXqcnZ2Nxx57DIGBgZg4cSJUKhWOHDmC7t276+MtULkernI4y01RWKpGRGymvsshIiIiImpWbaY5RGthc4iW88rGc1hzLB6Tgl3w5fRQCIKg75KIiIiIiOrU7ppDUMdwZ4grAOCfs8lYf+qmnqshIiIiImo+DE7UbAZ2tcPTo/wAAC+sP4tdF1K53omIiIiIOgQGJ2pWTw73gZXMCADw6E8n0XvJTvwbnaLnqoiIiIiI/hsGJ2pWZiaG2LRgsPS4oESNx38+hbjMfD1WRURERET03zA4UbPzcbCsceyxn05CVVSqh2qIiIiIiP47BidqEZN6ajck9rA1g9zUCJdT8zD0g72IuMZW5URERETU/jA4UYt4f2pPvDDOH3/NH4L7+nkAAJSFpZj+7VHEZxYgO79EzxUSERERETUegxO1CIW5MeaP9IWNhQm6OVnpnBv20V7c900ENJpOtYUYEREREbVjDE7U4qoHJwC4nJqHX4/H66EaIiIiIqKmY3CiFufjWLNZBAC8vuk8NkZyo1wiIiIiavsEURQ71XwplUoFhUIBpVIJuVyu73I6jd9PxCOvWI2t55JxKi4b3ZwscTk1DwAwyMcOtwU4YnyQM9xtzPVcKRERERF1Fk3JBgxO1KqKStUoLtXA0tQIr206j99PxKNiqZO/kxW2LxoKQRD0WyQRERERdQpNyQacqketytTYEApzYxgaCAif1hP7nh+J/l62AICY1FzEpObquUIiIiIiopoYnEivPO3M8cfcMAzv5gAAGP/ZQSQrC/HQ98dw38oIXM/Ix7FY7v1ERERERPrFqXrUJuyLScPsVSfqPL/r2WGws5BBEABrc5NWrIyIiIiIOipO1aN2Z4S/I14Y51/n+RM3sjHm0wMY/9lB5BeXtWJlREREREQMTtSGjPB3qPPcmYQcZOQVI0VVhD9OJrRiVUREREREDE7UhvRwVWDJlCDMCutS49zhaxnS9xsjEwEAuy+m4uN/YzgCRUREREQtzkjfBRBV9dBAbWhSiyJ+ORovHU/IKpS+P3tTiaXbLmHF/msAgC/2XMXLEwIwd7hP6xZLRERERJ0GR5yoTXK0Mq33fEVoqrB026WWLIeIiIiIOjkGJ2qTStWaWo8P8LZt5UqIiIiIiBicqI0aHegEAOjmZIlFo/0AAA5WMnxyXy+8fnv3Wp/TyTrrExEREVEr4j5O1GadT1TCw9YcCjNjpKmKYGthAiNDAxSVqvH+1otIyCrA3ph06fqx3Z1wMUUFG3MT/PzwACjMjfVYPRERERG1dU3JBgxO1G7tuZSKh1efrPP86jn9MMLfsRUrIiIiIqL2hBvgUqfgbW9Z7/nZq05g2teH8c/Z5FaqiIiIiIg6KrYjp3bL294C387si8j4bFxOzcO03m7YGJmIVFURzt5UAgBOx+egYM8VTAp20XO1RERERNSeMThRuzamuxPGdHeSHk/s6YLCEjUC39guHbuUkgtlQSmKytRwtJJBEAR9lEpERERE7Rin6lGHY2ZiiK4OFjrHHvrhGAa8vxvrTt3UU1VERERE1J5xxIk6pFWz+yE6SYUj1zLwy9F4aerej0duwMFKhj5dbCA3Zdc9IiIiImocjjhRh9TFzgITe7pgYk/dtU3RSSrMWXUCdy8/gohrmcgrLkOqqkhPVRIRERFRe8ERJ+rQBnjbwc/REtfS86Cp0nj/cmoepn97FLYWJlAVluKT+3rhzhBX/RVKRERERG0aR5yoQzM0EPDXgiGIfGMsfB1rti/Pyi9BmUbEW39FS8fyisuQnV+C84lKqDWdapszIiIiIqoDR5yowzMzMYQZDLF8Rm/EpOZiwZrIGtdk5ZegsEQNY0MBU746jKtpeQCAVyYG4PFhPq1dMhERERG1MQxO1Gn4OVnBz8kKRgYG+P1EPPbGpOucv5qWh7isfCk0AcD7Wy8xOBERERERgxN1PuODnDE+yBk7olPw6c7LuJ6Rj+IyDe748lCNay1MDAEAuUWlKCnTIDYjH3272HAvKCIiIqJOhsGJOq1xPZwxrocznv4tEn+dSQIACAIgVlnWZGZiiB3RKXhlwzlk5pcAAJZM7oGHwrz0UDERERER6QubQ1Cn182psmnErDAvvDjeX3qckVeCJ34+JYUmAPhgewzK1BrEZeajTK1p1VqJiIiISD8YnKjTe3BgF7jbmAEAJvZ0wZxB3hgd6FTn9RpRxII1kRj+0T7ctSJC51xiTiEWro3EufINd4mIiIioY9BrcAoPD0e/fv1gZWUFR0dHTJkyBTExMQ0+b926dQgICICpqSl69uyJrVu3tkK11FFZm5tg8/zBWDc3DP29bWFmYojvZvWFvaWs1usLS9XYHp0CADiTkIPsKqNRL6w7g81RSbh7xZFWqZ2IiIiIWodeg9P+/fsxf/58HD16FDt37kRpaSnGjh2L/Pz8Op9z5MgRTJ8+HY888ggiIyMxZcoUTJkyBefPn2/FyqmjsbOUoZ+Xrc6xIb520veLJwRI34vVtnaKzcjH9Yx8FJSU4XR8NgCguIxT+IiIiIg6EkEUq/8aqD/p6elwdHTE/v37MWzYsFqvue+++5Cfn48tW7ZIxwYOHIhevXphxYoVDb6GSqWCQqGAUqmEXC5vttqp40lVFWHKV4cR4GyFH2b3w19nkvDV3qu4nJqnc93sQV5YfeQGwrra4VRcNkrK1z09O6Ybnh7lp4/SiYiIiKgRmpIN2tQaJ6VSuy7E1ta2zmsiIiIwevRonWPjxo1DRERErdcXFxdDpVLpfBE1hpPcFAdfHIlVc/pDEARM7uWG8T2cpfNd7MwBAKuP3AAARMRmolRTOdL0yc7LSM8tRkmZBnsvpaGEo1BERERE7VabCU4ajQaLFi3C4MGDERQUVOd1KSkpcHLSXbjv5OSElJSUWq8PDw+HQqGQvjw8PJq1burYjAx1/xMJclMAAEyNDTBjgGeN66uP315JzcUnOy9jzuoTeGdLdIvVSUREREQtq80Ep/nz5+P8+fNYu3Zts9538eLFUCqV0ldCQkKz3p86l2HdHDCuhxOeH+uP+/t7wsSw/v+EYlJzsWL/NQDAL0fjW6NEIiIiImoBbWID3AULFmDLli04cOAA3N3d673W2dkZqampOsdSU1Ph7Oxc6/UymQwyWe3d0YiaytTYECsf6is9/umR/rj/m6N1Xn85NVfnsUYjokStQXGZBvti0jA+yBkyI8MWq5eIiIiImodeg5MoinjqqaewceNG7Nu3D97e3g0+JywsDLt378aiRYukYzt37kRYWFgLVkpUu+6ulYsIHxzoCScrU0wf4IlvD8Zi5f5Y/BWVpHN911e2wsTQAEaGAgpK1JgU7IKvHujd2mUTERERURPpNTjNnz8fa9aswebNm2FlZSWtU1IoFDAz025IOnPmTLi5uSE8PBwAsHDhQgwfPhwff/wxJk2ahLVr1+LkyZP45ptv9PY+qPOSmxrjhXH+yMovwWuTAiEIAgDgrt7uWLk/Fvkl6hrPKVFrUHH4n7PJeHl8ATSiiC52Fq1ZOhERERE1gV7bkVf8klndqlWrMHv2bADAiBEj4OXlhdWrV0vn161bh9deew03btyAn58fPvzwQ0ycOLFRr8l25NQaSso06PbaNunxKxMD8P7WS7Vea2QgQC2K+Oy+Xpjcy621SiQiIiLq9JqSDdrUPk6tgcGJWovXy/9I399YOgnLdl/BJzsv1/ucS0vGw9TYEKIo1vkXC0RERETUPNrtPk5EHUl/b+1+ZI8O0a7dmzfCBx/eFYy37+xR53MCXt+OpdsuocebO7Ax8iYA4MjVDOyNSWv5gomIiIioThxxImoh6bnF2HMpFdN6u8O4WtvyvTHaDXGf+PlUvfd4YIAn1hzTtjE/+OJIeNiat1i9RERERJ0NR5yI2gAHKxnu6+dZIzQBwEh/R4zt7gQnubZVvoetWa33qAhNADD0w704dCWjZYolIiIionoxOBHpiSAI+HvBEDw+rCt+engALi0ZDxtz43qf8/CPJ3AjIx83MvIx+avDWH/qZitVS0RERNS5MTgR6ZGj3BSvTAyEt70FTI0NcXuwq3TORWFa4/qSMg1e2XgOU74+jDMJOXh+3Zl675+dX4LwbReRlFPY7LUTERERdSZ63ceJiHQ9P84fycpCDPG1x+zB3lJnvjmDvXBXb3dM/uowjlzL1HlOZHw2jsZmIVVVhC525pg9yEvqyPfin2ex80Iqtp9Pwf4XRrb6+yEiIiLqKBiciNoQhZkxvpvVT3q85tEBWH/6Jp4d0w1Wpsb4+ZH++GznFSSrCpGQpR1Fmvr1EZ17pCiLsHhiIIpK1dhX3o0vLrOg9d4EERERUQfErnpE7dSvx+Lw6sbzNY4LAjDMzwH7L6frHI9+exwsZPy7EiIiIqIK7KpH1AncHuwKI4Oam+SKImqEJgC475sI3PnlIVzPyG+N8oiIiIg6FAYnonZKYWYsbbJbYfYgrzpbm59PVOHsTSW2nksGoG000ckGnImIiIhuGYMTUTv23tSeOi3M7+nrjl3PDsfBF0fi0EsjYW8pw8SezlgyuYd0zbW0POQXl2Hoh3sw84fj0nG1hiGKiIiIqC5c8EDUjnnbW+Dka2OQXVCC9NxiBLpo5+Z62JoDAI4uvg1G5RvwOljJMPeX07iWnoczN3OQqipGqqoYOQUl2BSZiLe3XMCbt3fHpGBXpKqKYGpsAF9HqxqvmV9cBgBcL0VERESdCptDEHUSV9NyMfqTAw1eZ2ZsiDKNBqVqEXeEuOLFcf5SECsp02DER3shCAL2vzBCCmVERERE7VFTsgH/ypiok/C0tYCJoQFK1Jp6ryssVUvf/30mCTuiU/DiOH/cyMxHaZmIJGURACBZWSQFKiIiIqKOjiNORJ3IpshELN5wTicc1cXN2gxGhkKde0C5WZth7eMDEZOSiyF+9jA1NmzucomIiIhaVFOyAYMTUSdz5FoGHvj2GAAgwNkKqaoiGBoYICOvWOe6K+9NgJGBgJk/HMfBKxn13nN6fw8EuSkwwt8Rbta1d/UTRRFqjcjpfURERNRmcKoeEdXJx8FS+v7BgV0wJdQNGlFEZHwOZpV32fNxsIBxecB54/buGPNp/WujfjueACABIe4K/PLoAJgZG0oBSa0RYSAAM384jtj0fGx5aggsTY2k+xMRERG1BxxxIupkRFGE9+KtAIB3JvfAzDAv6Vx2fgm+OxSLmWFecJKbSsfXHIvHKxvPNer+psYGkJsa4/FhXXF/f09M+/owAOByap50jYWJIXY9NxwuCu3oVG5RKVRFZXCzNkOyshCGBgIcrUxrvT8RERFRc2lKNuBf+RJ1MoIgYHIvV1ibG2NiTxedczYWJnhhXIBOaAKABwZ4IsRd0aj7F5VqkJZbjHf/uYg7vzyEy6l5OqEJAPJL1DgVly09vvPLwxi8dA9i0/Mw/rODGPHRPpy8kdXga5WU1d/ogoiIiKi5MDgRdUKf3dcLx18ZDXtLWaOfk11QKn3/08P98deCwbAyrX+2b2x6fp3nknO03fmKStW4nqG97s4vD0NZWIqCEjU+332l3nvvvZSGoDd3YO3x+Ma+BSIiIqJbxuBE1AkJggATo6b95z/Y1w4A4Gglw7BuDgh2t8bxV0bD36nmJrmNkZhTiKz8EgwM3y0dyyvfXBeoP3QBwLxfT6NErcHLGxo3hZCIiIjov2BwIqJGeWl8AJ4e5Yc/ngiTjpmZGGL5g72x4sHe0jErUyNMCtZOARzgbYvHhnpL5+aP9MGSyT0AAEk5hfj+UCxyqoxkVZWkLERxWd1t0xvaj4qIiIioObGrHhE1irW5CZ4d063G8a4OlujqYImvZ/TGqsPX8cm9veBha45l92v7zmyMTJSuHeHvCFWhNij9eyEViTmFdb6eKALnbiqRmFOIUYFOOHEjC+tOJmBiTxf097KFUOXa9Nxi7LmUirt6u7PdOREREbUIdtUjohZVWKLGZ7suY1g3Bwz2tcfFZBUmfH6wzuvv6eOO4zeydDbetTY31hmZGupnj6OxmShVa//35WlrjvisArwyMQC3BTjCx8ESgiDUuDcRERFRVeyqR0RthpmJIRZPDMRgX3sAgK+jpc4muT4OFvj8/l7S48UTAxHqYa1zj+rT+Q5eyYBaU/l3PvFZ2pD1/tZLGP3JATz1WySupet28iMiIiL6LxiciKhVGRsa4NBLI6XHowOdIDczlh7bmBvjubH+WDTaD3YWJnXeR1PPWPmWs8kY9fF+DHx/N66l5+Hno3F4Y/N5qDUi0nKLkFNQgn0xaVi84SwOXclolvdFREREHRvXOBFRqxMEAZvmD8bfZ5Lw9Cg/mBob4r6+HghyV0AQBHjYmmPR6G4Y290ZE5fVPa2vISmqIry28TwiYjMBAAO72mHer6dhYmggNZc4eSMbO58d3izvi4iIiDouBici0oteHtboVWVK3gd3B9e4xt3WTOfxigf7YHNUIradTwEA2FvKsGi0H/p0salz3VRFaAIqG1VU7ciXoirSub6kTIPLqbno4Spv0jopURS5roqIiKgD41Q9Imqz5KaVU/jmjfDB+CBnhHpaS8fuCHHBgwO7INBFDgsTwwbvt/NCao1juUVliErIAQDEpudh9qrjuP2LQ/jrTBLK1Bq8/OdZfLLzcr33VRaUYsT/9uGtv6Ib98aIiIio3WFwIqI2bYS/AwDgnr4eAIBh3Rykc36OlZvvvjwhAADgLDfFvudH4LuZfRHk1rjOmVO+OoxXNp7D5K8O48g17QjVwrVReH1zNNaeSMCy3VdQVFr3nlKbohIRl1mA1UduNOm9ERERUfvBqXpE1KZ981Bf5BSWwNHKFADg72QFY0MBpWoRA7raStc9OLALbC1k8He2hJe9BbzsLZBdUIIX1p+FnYUJbg92wY8RcXW+zppj8TWO/Xa88lhq+ZQ+e0sZLGRGOHkjCzsvpOKZMd1QWmXqX25RKayqjJQRERFRx8DgRERtmomRgRSaAG1jiT3PjUBmfgl8HCx1jk8KdtF57t193AEA/bxsYW1uXG9wasi+mHS8/Xc0+naxxe9PDMTdKyKkczKjysH7hKxCdHfVBqf9l9Ol9uv5xWVIURXp1AwA288nQ2FmgjAfu1uujYiIiFoep+oRUbvjYWuu01iiLoIg4J6+HvCyt4C1uQkcrGR1XmsgaEPalF6utZ5fcyweGhE4fiML+y+nS8e3nU9BsrKywUTFnlKbIhMx64fjGLx0DzZFJuLlDecw6uP9OHKtsv35ldRczP3lNKZ/exSdbC9yIiKidocjTkTUafzz9BAcuZqJbw7E4pEh3vjrTBL2X07HUD97LH+wD4wMBMiMDLApKgkAMLmXK4pK1dgRnYqY1FzpPrNXnZC+T8guQF5xmfT40NV09HRX4Ot9V6Vji36Pkr7/aEcMVs9WoFitxun4bOm4srAU1ua6+1ZdSFLByFCAk5Up7l0ZgTHdnfD8OP9m+3kQERFR4wliJ/trTpVKBYVCAaVSCbm8cQvHiahjSlMVYf3pm7ivrwfsLCtHo45cy0BSThHu7uOO8K0XsfJAbLO9pr2lDKbGBlAWlGJUoKMU0rYvGooAZzmuZ+QjTVWE7q5y9HzrXwDAksk98Ppmbce+2PcnwsCAbc+JiIiaQ1OyAUeciKjTcpSbYt4I3xrHB/nYS9/3cFPonAv1tEZkfA4AwNPWXJqa11gZecXS9xWhCQA2RyUhxbsIj/54EmUaEZ/eFyKdy8wvkb6/lp4HP6fKboIAUFSqxp5LaRjiZ6/Twp2IiIiaD4MTEVE9JgY5Q3NfL/wYcQMmhgaY3MtNCk5fz+iNX4/FwUVhhhAPa2w/n4yt51KgLCxt8uss33cNy/ddkx5vjKwMVVdS86TvIxNy4OdkhYKSMmhEwFJmhE93XcbK/bEYHeiE72b1vfU3S0RERHVicCIiqoeRoQGmhLphSqgbAO2I0QfbL8Hb3gI9XOUInxYsXTu8mwPCpwVjX0wafj+RgJNx2UjP1Y4wvX57d+QWleKzXVca9boHqjSguJiikr6/kKRCmVqD8Z8dRFGpGgdeHIkfy/eP2nWx5ga/t2JHdAr+PHUTH94dXGPdFRERUWel1+B04MABfPTRRzh16hSSk5OxceNGTJkypc7r9+3bh5EjR9Y4npycDGdn5xaslIhIy95ShgMvjISxkQBBqH2t0Qh/R4zwd0R8ZgF+PR6HR4Z4w9HKFIUlahSUqPFNlTVTVqZGyC0qq/U+FWLT86XvVx+5gb0xadIUwbf/vgBjAwMUQaPznIrlq+l5xViy5SJ8HCwwf6QvjA11m6n+fiIey3ZfxXNju2GEvyNsLUzwxM+nAABuu6/gzTt6NPInQ0RE1LHpNTjl5+cjJCQEDz/8MKZNm9bo58XExOgs3nJ0dGyJ8oiIaqUwb9w6Ik87cyyeECg9NjMxxCsTA+Ftb4HFG84BAJ4d0w1v/32hSa8fl1m5rqrqJr0AEJOSi63nkvH9oesY18MZHrZm+PuMdtqfkYEAf2c5rqXn4e4+7rCzMMFLf5bX8ccZBDhbYfuiYbW+DhERUWen1+A0YcIETJgwocnPc3R0hLW1dfMXRETUCob62UNmZIC+XjaYPcgLvTyskVtUhjmrT0CtEdHf2xbHr2fd0r3HfXZA+v7P0zfRw7XyL5m+O3QdOQXa9VdLt13CG7d313nupZRcnddVa7SjViVlGmyPTsFIfwdYsfkEERF1Uu1yjVOvXr1QXFyMoKAgvPXWWxg8eHCd1xYXF6O4uLKLlUqlqvNaIqLW4G5jjsMv3wYLEyMIgoBQTxsAwIEXR+L34/G4u48Hhn20t1leKzqp8v95FaGpwjtbao503bsyQvo+u0Dbze+5dWfw95kkzArrghAPayzfdw3PjOmGiT1dGlXDyRtZKCxVY6ifw628BSIiojbBoOFL2g4XFxesWLECf/75J/788094eHhgxIgROH36dJ3PCQ8Ph0KhkL48PDxasWIiotrZW8pgZmKoc8zN2gzPjvWHp515jevv6eNe45ixoYApvVylxw8P9q7z9bztLZpcY1xmAURRlKb6/RgRh2f/OIMraXmY9+tpZFVpk14XtUbE3Ssi8ND3x5GWWyQdVxaWQlVUivziMp0W7URERG1Vuxpx8vf3h7+/v/R40KBBuHbtGj799FP8/PPPtT5n8eLFePbZZ6XHKpWK4YmI2rwnR/hI7ck9bM3w/Dh/2FiYYEd0CkZ0c8CG04nYOH8wutiZw83GDD1cFZjY0wXPjPHDHydvIi4zHz9FxAEA+nvZwllhiusZ+fW9ZA3KwlJExGbWef5qWh76e9vWe4+q4So+swCOVqYoKlVj4Pu7YW1uDAuZEa6l5+HUa2Nga6Ht4KcqKoUAcFogERG1Ke0qONWmf//+OHToUJ3nZTIZZDJZK1ZERPTfvTQ+AC+M9ce19Dx42JrD1FjbWOKViYEQRRFv3dlD6ur3wrgA6XlWpsZ4ZIh25CnITYGjsZl44/buWF3esrwu6+aG4Z4V2ml6XR0scD0jH6IIPPDtsTqfcyMjv0ZweuuvaPx6LA6WMiN8dHcIXK3NpHNJyiIci83Efd8cBQAUKtXSuUNXM3BniCtKyjSYtOwgSstE7HthBEyNDXElNRfHb2Thgf6edXYyJCIiamntPjhFRUXBxaVx8+yJiNoTAwMBfk5WNY43Njzc29cD9/bVjrD7OlrWeV2AsxX6edli28KhiE3Px6RgFyzecK5Gx77qXvzzLHwcLdCnizY8nU9USgEtu6AU89acxnczKzfkTcgqwDcHrtV2K2m/q0NX05GQVShd7+dkhTGfahteWJgYSftpERERtTa9rnHKy8tDVFQUoqKiAADXr19HVFQU4uO1f1gvXrwYM2fOlK7/7LPPsHnzZly9ehXnz5/HokWLsGfPHsyfP18f5RMRtRujApxwV293OMll+PXRAdJxG3NjfDdLG24CXeSYFKz9i6gJQY3bG+/F9WcBAMVlavx+IkHnXEmZRgpEgDYImRrpruuqcCU1FwCw7VyKdCxJWYS4zMrphafispFbVIrJXx3Gx//G1LhHblEpopOUKFVrapwjIiL6r/Q64nTy5EmdDW0r1iLNmjULq1evRnJyshSiAKCkpATPPfccEhMTYW5ujuDgYOzatavWTXGJiKiSmYkhPr43RHq8ZEoQ/j6ThG8f6lvrvlTDujlgwUhffLn3ar33vZaej7Dw3cjKL4FBLSNhz607I31/PSMfOYWlNa4BtK3Q391yAetO3ZSOJeUU4np6nvQ4MacQB69k4ExCDs4k5ODevh7wsK1spPHgd8dw5qYSw7o54KeH+wMAfjh0HSmqIrw8PgAGBpX1nbiRheX7ruGN27tDZmyAS8m5GBlQuSdgdn4JMvOL4etYc8SPiIg6J0Gs2F6+k1CpVFAoFFAqlTqb6BIRka5kZSHCwvfUeu79qT2x6vB1XEnLq/V8c3jqNl9k5Zfg12Pav0Bzlpti4Wg/afPgp2/zxbNjtQ2D0nOL0e+9XdJzr70/ERpRhN+r2wAAq+f0wwj/ymDk88pWqDUiQjysoSosxfWMfCyf0RsTerqgTK3BbR/vR2JOIXY9O7xJHQlvZOTj+I0s3NXbHYYGXI9FRNTWNSUbtKt25ERE1Hqc5abS9y+ND9A598AAT2xfNAxfPhAKE6PKP0qMDQWM8G94v6bJVdqo1yUxpxBxmQXS4xRVkU5nwGV7ruLLPVeQW1SKvZfSdJ6boipCck5l+/NDVzLwyc7L+LN8RKtic98zCTnSPdeUr+n651wy4rMKoNaIOHGjaRsRj/pkP15cfxbrTiY0fHEzyi0qRUkZpygSEbUkBiciIqqVIAhY+VAfvDwhAHOHd0Wgi+7fxBkaCLg92BUrH+wjHQtwlmNFlcd18bCpnGJXV6+LGxn5iMvSbaH+zYFYncf/+/cyhnywFy/+eVbn+JSvDuNCcuXmv98duo5lu6/guXVnUNdEi4qQtjkqSTpWEaquZ+Tj3S0XoCqqfaphhYpA9vnuK1AWlOKfs8m4XL5+qyplHVMW6yOKIlJVRZj78yk8+3uU9D7yissweOke3Pll3R1miYjov2v3XfWIiKjljOtR2SSirpln3V0rA9X9/T1galyzAURvT2ucjs+BpcwI383qi90XU6Vzq+f0x6wfjutcLwjA6fgc6bGlzAh5xWW1vn5tISQ9txivbjxX6/XP/nGm1uOJOYXIKSjBkWsZ0rErqdqpiHd+eQi5RdrNej+7P7TW52s0lYEsWVmEkHf+lR6/M7kH7CxkmBTsgh8OXcc7Wy7gqwd6S804GuPfC6l44udT0uMXxvvDRWGGszdzoCoqgyolF4Ul6hobKxMRUfPgiBMRETXKff20rc1DPKx1jjvJTfHS+AA8O6YbHujvWetzV83pjy+mhyLyjTEY2NUOfbrYANAGouHdHPDFdN0wMqJb5XQ/U2MDBLnpjnaZ1RLOQtwVCHFXSI8zq2y+W9XGyMRaj6s1Ip75PQpFpZVT3nZdTMWhKxnILdKGtuPXs3AtPQ9nEnKQpirSGb3KyCuucc8Kb2yOxvw1p1FUqsY7Wy4AABZv0I6SFZWq8dGOSzh3U1nn8wHgg22XdB5fSNKOqFWdopeYUwAiImoZHHEiIqJGmTGgC7zsLGoEJwB4coSPzuO37+yBN/+Klh4rzIxxR0jluqZxPZzx9Yze6OulDVB3hLiin5ct3vzrPGYN8oK9pQwnbmQjr7gM9/b1QE6B7qjSz4/0R2xGvtQO3dhQwOo5/XHmZg5mrzpxy+9xb0w6AOC5Md3w89E4pOUW48HvKzcBTlIWYdTH+6XHd4S4Ytn9vSAIApKURTXuV13FCBYAWJubAACW7b6Cr/ddw1d7r2Hf8yPgYCWDhazmH8+eduaIrbLGKzpJhVGBTsjMqwyICdmF7ARIRNRCOOJERESNYmggYFg3ByjMarYvr27WIC9smDcIpsYGNRpLANr1UxN7usDRqrIBhbPCFCsf6otBPvbo5mSFzQsGY9Xsfnj7zh4o01SOqshNjeBfvmlvhduDXWFjYYIR/o649v5EfDE9FIKgbXAR1tWu1hqn1zE6NrCrLeaP9MXu54bDv5YNiKv6+0wS9sakoahUjYe+O1bvtQBwR5V1SA5WMgDA4auVUwNH/G8f5qyuDH4ZecVIU2kDmaba0qzI+GwAQGZ+5UjXzSyOOBERtRSOOBERUYvo7WmDc2+Ng7Hhrf0dnY+DJXwcLAEAxVWmz+1/YSSsTI1hYVL5R1hhiVr63tBAkEawbC1MUKrW4N8LKfjfjstIzCkEAJx5YyzkZkb452wSVOXT8FbN7ocd0SlYNLobDAwEWJkaY1g3e8TU0tyhqv0x6UjPLUZuHWuw6pJdUIKb2QU1nnf8ehZEUUSZRsTkLw+jqFSNfS+MQGb5VEB/JyvEpOZi/+V0JGQV4OCVyuD1+uZobI9OwU8PD9Bphy6KIpbtvorsghK8eUd3CHV15KgiK78ERaVquFqbNel9ERF1VBxxIiKiFnOroam6haP9YGwo4InhXWFjoZ3iVnVD295drGs8x1lhChMjA1jIjDA11B1393EHAAS5yaEwN4YgCPh+dj90sTPH97P6YmSAI5beFQxnReUomF+1EScjAwEDvG11jh28moEV+3W7/TVGbHo+hnywF7Hp+TXOZeSV4FJyLhJzCpGZX4LT8TnSlLz/3ROCAd620IjAfSsjdIITABy+molkZaH0uLBEjU1Rifh012WsPnID8Y0clZry1WEM/mAPNkbebPjiKk7HZ+OPkwl1di8kImqvOOJERERtXrC7Nc68ORbmJrp/bO18Zhj2XErDzDCvBu/x5AgfeNiaY3iVxhP9vGyx/4WRdT7Hz9FS+n7lQ30Q7K7AldQ8HLte2QWwIvg4Wsmwak4/TFqmnY736BBvfHfounRd5OtjYCEzws3sAtxWZZ1UbeKz8nWaRZy4niVNybOzNMGoQEccu55V57qqtNxiqArLsOC30zWCWUZeCbrY1dzUt0ytwcErGdh2PhmOVqZSwPrfjsuYGupeb71VTfv6CADAwsSoSV0DiYjaOgYnIiJqF6qHJkA7IlR9VKgupsaG0qhTY1W990BvOyjMjVFaVjmS4qIwRXJ5eFk42g9eVQLJc2P9kawswj/nktHb01oaKbMt/2d94jILcKpKO/Y9l9JQqhal51dd31Wb5JwifHMwttbRrKxq3QYPXcnA//6NQZCbHL8cja9xfam67o11RVHEW39Fw9rcBM+M6SbtYwUA89echq/jMPg7s1kFEXUMDE5ERER1sJQZ4c8nwyCKgMJc2xTD084cS6f1hLW5Mf4+m4x/ziYDAMK62sFCZoTfHx8IAwMBZiaGeGdyD/R0V2BabzfpnnLTyuYaowMdcVuAE16ptudU9b2mKjbztZIZwdTYED1cFTrnDQTg0/t6YeHaKADa0FKXhKwCfL7rCrrYmSPITS51DYxKyKn1+tyiMmw5mwRHK+3UR7VGlNrJX8/Ix48RcQC0zTaq7/V1/zcR+OTeXhgZ4FhnPf+FRiPirzNJ6O9ty7VYRNTiGJyIiIjq0adLzdGd+8s78l1Irmwc4W2vHW0aUKWLn52lDHOH67Zqr7o2a4S/Ix4Y4IkPd1yq0XK9gomhAUrKR336l6+vMjEywO3BLthSHtquvT8RgiAgMj4Hq4/cqPf9VOwjVZc+XWxwKi5belxYqsaCNZE613z5QChuD3bVWS+1/3IaAl1099vKLijFnNUncOGdcdKIYYqyCO/+cwFuNmZYOMpPZySxYhPhip9RmVqDHw5fx6rDN/DMmG64t6+Hzv2X77+Gj3bEoIerHP88PbTe99VUqw9fx9bzKfj2ob5SaCaizo3NIYiIiG7RI0O8MdTPHkumBDWqU12FZ8d0w2BfO2nq4Oo5/TE11A1/PhlW49r+VZpRvD25h/T9e1N6YkKQM96f2lN6bXktreLXPDoAN5ZOwhPDujZYl4mRQZ2bGFf11l/RKCxRI6FKcNoXky5NW6wuKafy+JItF7DlbDJW7o/Fn6e0jSc2RSbiWGwmJi47iK6vbMW4Tw/gfKISb/4Vjfe3XkKysggbTtdsUvHNAW1TjujyzYBvRUFJGcqqTUfUaES89fcFHL+ehc1nat8wucKmyER8uvMym2EQdQIccSIiIrpFCjNj/PzIgCY/7+lRfgD8pMe9PKzR675eAABzE0MUlLdXf+uO7ujhpoAgAM+M6QZ3G/PK1zY3xvIH++jct2p0szAxxLyRvhjkaw+gcWurBvvYoatDzcYR1WXklSDwje1wlld2IDx0JQN9y9dedXWw0FlfdS09D+FbL+JobCbyq7SOj4jNhI+jJRb9HqVz/5jUXNz+xSHYW8qkY1XDFwCoNSKUhbWP0gFAfnEZ3vwrGhN7OuO2AKdar1EVlWLER/uQlV+CBwd64oVxAVCYGeNyWuVIYn2dITUaUaq9r5cNhvo51HltYxWUlOG34wmY3MtV5/0Tkf4xOBEREbUhfy0YgqtpuRgfVNmRrrHhbMYAT/x7IRV39XbDo0N1R5hMjQ2l73u4ymuM0vTpYoP/3ROChsZNhvrZSy3QU1SVYSa3uAxLyqcBBrspdILTtwdicbLK9D8zY0MUlqqxIzoVZsZ1/yqSkVe5uW+yshBqjSjtTxWdpNS5dum2S3hpvL80+rb6yA2sP3UT60/dxI2lk2q9/7HYLKlZxi9H43H8ehYG+djrhMyiUnWtzwWApCpt36+l5TVLcFqx7xqW7bmKNcfisPu5Ef/5fi3pekY+Xt14DguqBPS6lJRpYGLEiU7UvvETTERE1Ib4OlrqhKamcJSbYtvCoTVCEwCdDXGn9dZOEbS1MMGy6aE4+OJI/PnkINhZymBXx8jUwRdH4qO7g7F6Tn/cGeKqc67qKJUgAON6OOucrxqaAOD5cf6wtzSBWiPiz1qm4NWmVC1i8Yaz2H4+GbHpeTX2r1qx/xrOVmnhfjO7MtQUl9UMP8nKQjxTbaTrcmoeVh+5gU92XpaOqQorNyguU2uwaG0kvt53FQBwrUo4PJuo1OkqeKs2RiVK976UUvsUxBRlEYZ8sAcf/xvzn1/vv3jm9ygcuZaJB747Vu91f5xMQI83t+Pf6JRWqoyoZXDEiYiIqBOY1tsNuy6mYnSgE+7u4w5DAZgY7AJHK1Od6+paq+Vhaw4PW+1UwWXTQ/HJvSFYf+omcgpL8fBgb5y9mYOcglL4OVnWuk8UoF3blaoqwowBnvCyM8cjP55ssG5BADxtzRGXWYA/Tt7EHyfrDlqn4rJhbW6MJ385LXUiBIArqXkIctPtRPjG5mjkFZdVv0UNqqJSfLj9ElJURRjXwxmbopIAAE8O90Fsep503YbTidhwOhFjuzvhmTHdEOgix+8n4nHiRjaWTusJAPjh8HXcFuAIBytTbD2XjAlBzrA2N8G6kwnYEZ2CScEu0kbHAHA6LgcBzroNNwBgU1QibmYX4os9V/HwYG+p1X1zSFMV4fiNLEzq6dLgur3EnMJaj4uiiPf+uQg7SxmeHOGDF9efBQA8v+4MzlYL1fpyIyMfmfklUofIzigzrxgaEXCw4pTQxmJwIiIi6gTMTYywek5/6fHswd7/6X5GhgZSd0EA0vqmunSxMy9f26U10l+3Rbm/kxViUnOrPw2iqN2IOC6zoMa56vZdTsf5RKVOaAKAC0kqnL2pxD/nkvDQwC7YcDoROy+kNng/QDu68885bfdCocoqsoy8ElyrEpwq/HshFSKAJ4Z1xUt/atvMj/B3QEGJGu9vvYT3t17CtFA3bIhMxNZzyfh6Rm+8UB4sdl1M07nX9YzK+5eqNXjwu2NwVpjCWVEZdv8+myRtAF2m1uDTXZehKizDO5N7QBAEnLyRhS1nk/HCOH9YyOr/ta9UrcGwj/aiqFQD2UxDjOle+9qwCoZ1BKtLKbnS5s8zBlZ+RqpOF9W3O748hNyiMmyePxghHtb45WgcPt99Bavn9KvR7r8jKlNr0OfdXQCAS0vGt6l/N20ZgxMRERHp2LZwKP4+k4Sv91275XusfKgPfj0Wj6OxmSgp0+Du3rqbD1dty25hYogdzwzDQ98fqzEFDwAWTwxE7y42GOnvCFEEJi47KJ376eH+ePnPs0hSFuHA5fRaa/nh8HVcStGGssNXM2u9RhC0Ia26itAEAMdvVD43MacQ19K0U/X+d08I/BwtMfmrwwCAnRdSdYJZdkEpkqqMzmyI1E7HO3glA2cSdNdqVXU9ozIsxqTk4tj1LABAV/vKEb1jsVlScPpq7zV8tVf77+y2AEcIAvDYTydRqhahKirFJ/f2kp6XW1QKcxMjnSmcm6OSUFSqKX8PKcjMK0ZSTiFCPW0QnaSEg5UMHjbmUBWVYXyQc419uypUDbnnq0yfNDMxxJpj8Rjd3VFnpPOPkwn45kAssvJL8PL4ANzbT7ftfHMrKdMgt0g72rgpKhEhHtZ4bdN5ANq1crfS8KU1iaJY52jgxWQVvj90Hc+M6Qa3evY2q9pYJTO/pN5rqRKDExEREekIdJEj0EUOmZEhPt11GS+ND2jyPcb1cMa4Hs5IyinE1nPJmDGgS41rlk0PxeI/z+KbmX0B1D1lyMfBEvNG+NZ6blg3Bxx++TYM/XCvzrqmqipCU20WjvLDM2O64bGfTjY4CpWQVXn/xOxCXCnvvufraIkQD2tsnDcIU78+UuN5eUVlKCypvcnE6fjsWo8DlSNOysJS3MyuDCOxGZVrq6o+/1SV7+esPqFzrw2nE/HelJ4wMzHE/3bE4Mu9VxHgbIWuDha4nlGA72f1RcS1ymDY0LTIP58cpPPL+/7L6Qh0sYKjlSliq4yU7aiyrikuswCvbDyHlQfMMaWXG8Z0d0KQm0KaygcAL/55FtfS8zDC3xFhPpV7ojWn1CpNTaIScnRayZep616nptGIuJyWCz9HK53A2RiFJWokZBegm5NVo64XRRGP/XQSysJS/PbYQBiVd3c8ezMHs1edwKNDvWv9b+LeFRHILS5DYnYhfnt8YJ33rxqcCksanrJa3d5LafCwNYevo2WTn9uesTkEERER1erpUb7Y89xwzB3e8B5QdXG1NsOjQ7vCzKTmVKA7Q1wR/c54DC7vyPbCOH/08rDGx/eE1HvP6m26BUHAZ+Xt3Cu8OjEQ704Jwkj/+jvdPRSmDXQBzo37hbbC/DWnkVG+HqmiOUb1DYArfLD9Eg5eqX00rKIRhZddZav5Hq7a+8RnFSA2PQ9h4bsx95fTtT4/WVkk7YeVlV9c6zUVFm/QBpRN5Q0oLqXkYuu5FFxMVmHBmtOITKg7xFW3Yv81GFT5LXLWD8fx1JpIXEnNxckblfepOmJXIS6zAJ/vvoLbvziE/FrWma08EIsnfj4JZR2bQoui+J8acVTdb+zcTaVO4K5vOuN3h2Ix/rOD+Grv1Sa/5tJtFzH20wNYf6pxzVCKSjXYdTENJ25k42xi5ajdnV8eRlZ+CT7cXntjkNzyn2d9gRwAcqoEp4rRt4Z8dzAWK/dfw5FrGZiz+gTGfXag1uvO3VRi2Id78feZJJ3ja47FY/aq441aW9hWMTgRERFRrQRBQFcHyyZt7vtfuCjMsGn+YNzVxx09y5s59HSrud7k25l94KowxcqHKvex6utli+i3x6G7ixyTerrgsWFd8eDALujpbl3va9qaaxsrzB3ug3v7usPsFtZ6yE21Gw/Xt06kage+2owOrFxPFOgih7GhgFK1iM93X5H29arKzNhQ6m743LozeG3TOaSq6g9O26NTkF9cpjNtsEJUQo5OC/mGnE9UwqDa5+LY9SyM+fQA9lyqXKuVUaXZRW0qWthXpyoqw9xfTmFTZCIKqoyIFJao8fDqE+j77s56m1McvJKO7PzaXzu5Shv5Mo2IL/ZcqXzd8kChLCjF239H40qVdXfvb70EADpdF+tz4kaWNFL4Y0QcAG2DjIaUqTU6I0Lx5VMfU+rYYLo21f/dVFc1lNYVnM4k5OCh74/hQpIK2fklePefiwjfdgk/HrkBAHWG15c3nEV8VgGe+i0SR65mYOu5ZOyLScMrG89hX0y69Pz2iMGJiIiI2pyVD/XB48O6YvmDvWucC/W0wZHFo2q0PbeQGWHrwqH4akblc/xrmRpVtZNaxVorC5kRPrw7BFN7u9VbV4i7bpAb1q1pezd52Na+lmSwX+U+SHYWJlIDiL+q/a19hTtCXPDJvSHSSNUvR+ORnlt7cDI0EGBvaYKiUg16vLkDtf2+W3HM3lKms4aqLsnKokY17GjI2hMJdZ6LiM3Eot+j8P7Wi9Kxn4/ewN6YdGQXlGJjHa3sl+2+ioe+P45n/4iSjiVkFWD2quM4cjVDZ8QJgM6UxIvJKny19ypC3vkXqw7fwL0rIwCgxkiRKGrXjdVlb0wa7lkRgSd+PlXj3Be7K4PajYx8DP1wD747GAtAOyrY591dOo1SYlJzsedSKgaG75aOGRoI0GhEiKKIT/6NwS9H43SmHBoIwOXUXKg1IhKyCrDq8HWd6aI5hZWhsq4RoPu/OYqDVzKwYM3patMvK6e0irUsDKz6c3ngu2OY9+tpzF5VOXW0anBtbxiciIiIqM1xtTbDKxMD4W5j3vDF9RgZ4ICwrnZ4bKg3hvjaY3SgE36Y3Q/Duzngnck9alxvbWZc49iTI3yk70cF6naaCy9vNV7h2THdYGwo4PP7e2Gwb801OrWNoAGAU5VmCdbmJnBVaANW9d9L7+7jjun9PfHqpO4wMjTAi41Yf2ZtZixNh2yIo5UM/zw9FE8Mqzk9U2FmjMUTAmBs2PwjkA8M8MSy6aHo28UGy6aH6pzbHJmEC0kqlJRppIYcAPC/fy/j9i8OYtrXh6WAmZlXjE93aUeE9sak46Mdl3DuphKP/XQS+2LS8ciPJ3GpvOtibaOLucVl+GhH5TS47IJSHI3N1BkpsjAxxHv/XETwW//iVPkeZcrCUsRl5uPPUzfx/LozmFMeFKKTVHjg26M6r/HD4es4n6hEQlYB3v3nAhKyCvHuPxchiiJ+O54AZWEpfijvSghoG4PM/zVS5x5qjYjfTybAe/FWLNtzFa9tOo/rVda+5ZeoMfbTA/hoRwyWbruEt/++gCd/rQxxOVVGnLKqjMwVlaqlTZ8Ly/8Zm5Gv83OvqrbRUPN6NrUGgIJitc6avfaEzSGIiIiowzI3Map1kfyPD/ev5WpAZlT5y/TLEwLgYCnT+cVydKCTNFVrSi/XGt3Inh7lh7nDfWBiZIDJvdzw3cFYvPtP5YhJxV5YALD/hRF4669ozB7sDTvLyr2YFGbGdXY5mxnWBcFVph9O7OmCp0f5YVmVUYwKXz4Qih+P3MC8Eb7o4SrH5qjK0auuDha1Ts2zszSBmYkh7u7jjpUHYnXOnXlzLADtmq2GvHF7dyzff63OUbDqerlb484QV9wZ4lpjw+Lc4jJMXHYQ9/X1QGa16XfnE7UhKCMvBneGuOJAtbVkVTsNAtowULEX13tTg/DsH9pAdHuwC7acrbkeC4BO8wpAG0oq2q0/83sU4rPqDwFHqjTdMDM2RHZBKW7/4hAMBOiM/p2p0oGw6mcuOklZ6ybOizec03n8+8mao3cr9le+930x6cjKL4GthYlOcHpt03m425ihv7ctxnxyAJYyI/zz9BDpvMzIANcyarbeB7SjS9XXhTU0s3dDZCI2RCZKjVnaE444EREREZUzMar81WjucB/c1cddZ+pRoEvl1L+yOtZ4VL3Ho0O7Ymz5fkhv39lDJ5h1sbPAqjn9MbybA2zMK4OTgaAdcavgaCXD3udHYMWDvXVCU4WKZhLVTerpgnVzB2FkgCMc5aaYXmXfrSF1jEDZlW+mW9+mqA31ZZjSyxVzBnuhi23jRwv7elVOn5QZGeKZ0d0wwt8BU0Mrp07+fjIBuy7W3vkwPqsA288n45nftUHIroFNgQf52GFab3eM7+EMY0MBc4f7wMa85mhjxb0B7WhfXecaq+rIX/Wf4/tVAva5Kg0hUlXFDf7MAWDl/tgGrzl8NQOHr2borKECgNmrTmDg+7uRmFOImNRc7I2pDKDFZRr8UW1KZUU4qlgf9enOy3j5z7MoVWuQkde4sPz57it1biHQVjE4EREREZWzNK05Gee2AO1mvW7WZjqNMrrYNS4YLL0rGOvmhmHWIC841hFIqoYtQQDcbCqDk4vCFN72Fhgf5FLrc30cLHSuXfPoABx++bYaTT2q1juijm6DduUdCxXVpixWDSLVzwHA5F6u0vdBbgoIggB3m/r3Bqra8dC72rqqhaP9sHpO/zoD3uxBXgCAWWFd0M1J2xK7aufBT6t1WazOo3wK6Gf398Lhl29DkJsCXewqa/huZl9M76+7n9Si0X5wktcdKBtj0Wg/jApwxOf398Jn9/XC6MDKjaCP38j6T/dujKd+i8SM745hdS0NGlRVmkSs3K+7h1t2lREqEyMDeJaH4nUnE7DrQio+330Fa08k4LuD1xtsCFJhkI8dhvo1bgppW8HgRERERFRuWqgb/J2s8MgQb+lYqKcN/l4wBFue0k5fWj2nH6aFuuGJ4T513UaHrYUJ+nnZAtCOWozp7lTr+qopvVzhaCXD+CAXnfbojnLTGtdW5WVnAVl58Ap0kWOQr32tU/1sq4xqhdTRbbBiyqAgCAjrql2jNaa7E9Y8VjndsWo3QwDYMG8QHhtauSaqYrTKpYFNVT+fHooFI33x3cy+dXZu7F2lkUdVD4V1wa5nh+HVSd0R6qF7ze+PD8Swbg5YUsvPuIJTefMNU2NDaTNezyojZMEeCp2fkYWJIdyszRq15m7J5B5YMNIXPz7cH1525rg9WBt47+7jjiA3Bb6f3Q+Te7lhSqgbvpvVr8Y6uYa4WWun1X35QGjDF9+ik3G1tzN/aXwAdj87HFblf8Hw7cHrePSnk9L5T3Zq14cZGwp4ooFtDJY/2KfVOnY2F65xIiIiIipnITPCjmeG1Tjes0o3vRH+jhjh71jjmsYwNTbEt+Ub/lb32f2hUGtEGBoI6OFa+Xq17XVUlZGhAY68fBtWHb6B8UHOdV5XMR3O3MRQGlmqzt6i8vgvjw5AcZka5ia6vy4O7GqHS0vGY8pXh9HDVYHenjY6rb8rfql2VVQGvgHetjh2XTui8tqkQBgaCJCbGuP5cf71vjdvewssHOWHz6ut4bK3kEFRPrWudxdraX1PoIscA8oD30NhXjifqJLOdbW3kDYPrm3kyEJWOY3S3kIGnyqbu3ZztoIgCAh2V0gNIeryUJiX9P2+F0YCAB4bmlPnZrGhntbS948N9ca3BysbQxgZCDWmhA7r5oDwaT11Wor397bF8fKf77BuDtIUuABnq3o3gG6qucO7QhAEqQV/daXlGwgHu1tjwUhfZOaVYP2pmxAEYM2jA/Hkr6ek9VW1jVy2dQxORERERG2EYXl79KpT9+qa3leVnaWswRDS1cESW58eCtvyaXfe9ha4npGPeSN88PU+7dQsdZU2foYGQo3QVMHU2BDbF1UGTOsq64Osy0e2XBSVI06PDPFGTkEpgt0VeHRo0zZUfmZMtxrBSW5WWVeoZ+WIU/WpX7MGeSFJWYheHtawszDBW39r942q2sWwet2Atk29j0Nl0JkxQLtRci8P6xrPMzIQMKa7E7adT4G9Ze1rq0JqeV4FP8fK0cVpvd1xW4ATppd34hsd6IQQD2udhhxqjQaA7s+gh6tcCk5v3B6IF9aXorenDe7r54EHvj0GjSjqNJyoi8zIAA5WMp1NgauqGCGyqmVKa1XPj/WHlakx/ndPCJZMDkJBSRnsLGWwMDHSaUzR3jA4EREREbVB6+eG4ftD1/HqpO7Nds/uVRpJ/P74QBy6moFJwS74/UQCMvNL0M+r9qlxDREEbQv2a+n5CC0PCS7WleHERWFW60jerao6xcu3SsDpXz4lskJ3Vzl+fmQAACCiSnc7p1qmP84Z7IWNpxNxR4h2ap2thQleHO+PgmI17irf36v6tEBA28hjyZQguCjMaqyLagxDAwF/LRiMzPwSBLroNvoIcpNjfJCzTnCq2Gi56s8gxN0an94XguJSDXwdrbBx3mDp3MnXRuPPUzfxXHlLdQ9bMwz0tkMPVzne+vsCBAF4bkw3xKbn48kRPvh89xUpOD11my8Ky7sIVt3DrK4RJ0A7JTHMp7IVv5mJIcxMtDVP6+2GL/ZcbdReYW0RgxMRERFRG9TXyxZ9qwWB5uQoN8W03tpOcf8+MwwpqiL4OtbcMLixJvfS3TzYtcqIk8y4+ZbVD6+26bCBgYBVs/vhaloeRgXWPYXSz6kyYNXWNdDRyhQRi3Wbaswb4atzjaedOVbN6YfLKbkI36YNM13szGFvKcMbd9x6wK3eLXHnM8Ow82Iq5gzyhpmJIXYsGob1pxKw80KqTk0/P9Ifx2KzcEeIqzRaWRvPKo1BPGzM8dE9IQCAkQGOcJKbSmEMgM5Im72lDPf29UAXewuMqbKHWW1NVN6dEgQvOwsM7Fr3Z3b+SF84WskwMuDWprrqG4MTERERUSdnZymrc93TrbI2N4aVzAj5JWVSF7tb9dyYbvh452V8dl8vTAqu2V1wZIBjg7+M21vK8OgQbxSVqevsjteYZgUj/R3haWsuBSe//xA26+LnZAU/p8r7+jtb4dVJ3WuMPg71c8BQv9o7JFbV29NGmppZ0fQDgE4nwQpdq3RptLeUwczEEA8N7FLtGkvp2vCpPXE5NRd393HXCWC1MTU21FkD1t4wOBERERFRsxMEAUdfGYVStUaaqnWrFtzmi5mDvP5zQ4HXbm+eaY9Vp6pVDRptlaGBgL+fGoJdF1IxurtTvdd2ta864lT7mq0H+nuip5sCgS5WkBkZSg05OjoGJyIiIiJqERay5vlVUxCENtWFrWpzBFfr+tvFtxWWMiNMCXVr8DrvKkGwrn9/hgZCrY0yOjoGJyIiIiKiJjA1NoSVqRFyi8rQ2/PWGmq0VZYyI0wNdUNidqHOfmIECKIoig1f1nGoVCooFAoolUrI5fKGn0BEREREVE1WfgmKy9Q6bdep/WlKNuCIExERERFRE1Xsh0WdR/P1hiQiIiIiIuqg9BqcDhw4gDvuuAOurq4QBAGbNm1q8Dn79u1D7969IZPJ4Ovri9WrV7d4nURERERE1LnpNTjl5+cjJCQEX331VaOuv379OiZNmoSRI0ciKioKixYtwqOPPoodO3a0cKVERERERNSZ6XWN04QJEzBhwoRGX79ixQp4e3vj448/BgAEBgbi0KFD+PTTTzFu3LiWKpOIiIiIiDq5drXGKSIiAqNHj9Y5Nm7cOERERNT5nOLiYqhUKp0vIiIiIiKipmhXwSklJQVOTrq7HTs5OUGlUqGwsLDW54SHh0OhUEhfHh4erVEqERERERF1IO0qON2KxYsXQ6lUSl8JCQn6LomIiIiIiNqZdrWPk7OzM1JTU3WOpaamQi6Xw8ys9s3HZDIZZDJZa5RHREREREQdVLsacQoLC8Pu3bt1ju3cuRNhYWF6qoiIiIiIiDoDvQanvLw8REVFISoqCoC23XhUVBTi4+MBaKfZzZw5U7p+7ty5iI2NxYsvvohLly7h66+/xh9//IFnnnlGH+UTEREREVEnodfgdPLkSYSGhiI0NBQA8OyzzyI0NBRvvPEGACA5OVkKUQDg7e2Nf/75Bzt37kRISAg+/vhjfPfdd2xFTkRERERELUoQRVHUdxGtSaVSQaFQQKlUQi6X67scIiIiIiLSk6Zkg3a1xomIiIiIiEgfGJyIiIiIiIga0K7akTeHipmJKpVKz5UQEREREZE+VWSCxqxe6nTBKTc3FwDg4eGh50qIiIiIiKgtyM3NhUKhqPeaTtccQqPRICkpCVZWVhAEQd/lQKVSwcPDAwkJCWxWQY3Czww1FT8z1FT8zFBT8TNDTdVWPjOiKCI3Nxeurq4wMKh/FVOnG3EyMDCAu7u7vsuoQS6X83801CT8zFBT8TNDTcXPDDUVPzPUVG3hM9PQSFMFNocgIiIiIiJqAIMTERERERFRAxic9Ewmk+HNN9+ETCbTdynUTvAzQ03Fzww1FT8z1FT8zFBTtcfPTKdrDkFERERERNRUHHEiIiIiIiJqAIMTERERERFRAxiciIiIiIiIGsDgRERERERE1AAGJz366quv4OXlBVNTUwwYMADHjx/Xd0mkB+Hh4ejXrx+srKzg6OiIKVOmICYmRueaoqIizJ8/H3Z2drC0tMRdd92F1NRUnWvi4+MxadIkmJubw9HRES+88ALKyspa862QnixduhSCIGDRokXSMX5mqLrExEQ8+OCDsLOzg5mZGXr27ImTJ09K50VRxBtvvAEXFxeYmZlh9OjRuHLlis49srKyMGPGDMjlclhbW+ORRx5BXl5ea78VaiVqtRqvv/46vL29YWZmBh8fHyxZsgRV+4rxc9O5HThwAHfccQdcXV0hCAI2bdqkc765Ph9nz57F0KFDYWpqCg8PD3z44Yct/dZqJ5JerF27VjQxMRF/+OEHMTo6WnzsscdEa2trMTU1Vd+lUSsbN26cuGrVKvH8+fNiVFSUOHHiRNHT01PMy8uTrpk7d67o4eEh7t69Wzx58qQ4cOBAcdCgQdL5srIyMSgoSBw9erQYGRkpbt26VbS3txcXL16sj7dErej48eOil5eXGBwcLC5cuFA6zs8MVZWVlSV26dJFnD17tnjs2DExNjZW3LFjh3j16lXpmqVLl4oKhULctGmTeObMGfHOO+8Uvb29xcLCQuma8ePHiyEhIeLRo0fFgwcPir6+vuL06dP18ZaoFbz33nuinZ2duGXLFvH69eviunXrREtLS/Hzzz+XruHnpnPbunWr+Oqrr4obNmwQAYgbN27UOd8cnw+lUik6OTmJM2bMEM+fPy/+9ttvopmZmbhy5crWepsSBic96d+/vzh//nzpsVqtFl1dXcXw8HA9VkVtQVpamghA3L9/vyiKopiTkyMaGxuL69atk665ePGiCECMiIgQRVH7Py4DAwMxJSVFumb58uWiXC4Xi4uLW/cNUKvJzc0V/fz8xJ07d4rDhw+XghM/M1TdSy+9JA4ZMqTO8xqNRnR2dhY/+ugj6VhOTo4ok8nE3377TRRFUbxw4YIIQDxx4oR0zbZt20RBEMTExMSWK570ZtKkSeLDDz+sc2zatGnijBkzRFHk54Z0VQ9OzfX5+Prrr0UbGxudP5teeukl0d/fv4XfUU2cqqcHJSUlOHXqFEaPHi0dMzAwwOjRoxEREaHHyqgtUCqVAABbW1sAwKlTp1BaWqrzeQkICICnp6f0eYmIiEDPnj3h5OQkXTNu3DioVCpER0e3YvXUmubPn49JkybpfDYAfmaopr/++gt9+/bFPffcA0dHR4SGhuLbb7+Vzl+/fh0pKSk6nxmFQoEBAwbofGasra3Rt29f6ZrRo0fDwMAAx44da703Q61m0KBB2L17Ny5fvgwAOHPmDA4dOoQJEyYA4OeG6tdcn4+IiAgMGzYMJiYm0jXjxo1DTEwMsrOzW+ndaBm16qsRACAjIwNqtVrnFxYAcHJywqVLl/RUFbUFGo0GixYtwuDBgxEUFAQASElJgYmJCaytrXWudXJyQkpKinRNbZ+ninPU8axduxanT5/GiRMnapzjZ4aqi42NxfLly/Hss8/ilVdewYkTJ/D000/DxMQEs2bNkv6d1/aZqPqZcXR01DlvZGQEW1tbfmY6qJdffhkqlQoBAQEwNDSEWq3Ge++9hxkzZgAAPzdUr+b6fKSkpMDb27vGPSrO2djYtEj9tWFwImpD5s+fj/Pnz+PQoUP6LoXasISEBCxcuBA7d+6EqampvsuhdkCj0aBv3754//33AQChoaE4f/48VqxYgVmzZum5Omqr/vjjD/z6669Ys2YNevTogaioKCxatAiurq783FCnxKl6emBvbw9DQ8MaHa5SU1Ph7Oysp6pI3xYsWIAtW7Zg7969cHd3l447OzujpKQEOTk5OtdX/bw4OzvX+nmqOEcdy6lTp5CWlobevXvDyMgIRkZG2L9/P5YtWwYjIyM4OTnxM0M6XFxc0L17d51jgYGBiI+PB1D577y+P5ecnZ2Rlpamc76srAxZWVn8zHRQL7zwAl5++WXcf//96NmzJx566CE888wzCA8PB8DPDdWvuT4fbenPKwYnPTAxMUGfPn2we/du6ZhGo8Hu3bsRFhamx8pIH0RRxIIFC7Bx40bs2bOnxnB0nz59YGxsrPN5iYmJQXx8vPR5CQsLw7lz53T+57Nz507I5fIavyxR+zdq1CicO3cOUVFR0lffvn0xY8YM6Xt+ZqiqwYMH19jm4PLly+jSpQsAwNvbG87OzjqfGZVKhWPHjul8ZnJycnDq1Cnpmj179kCj0WDAgAGt8C6otRUUFMDAQPdXRUNDQ2g0GgD83FD9muvzERYWhgMHDqC0tFS6ZufOnfD392/VaXoA2I5cX9auXSvKZDJx9erV4oULF8THH39ctLa21ulwRZ3Dk08+KSoUCnHfvn1icnKy9FVQUCBdM3fuXNHT01Pcs2ePePLkSTEsLEwMCwuTzle0lh47dqwYFRUlbt++XXRwcGBr6U6kalc9UeRnhnQdP35cNDIyEt977z3xypUr4q+//iqam5uLv/zyi3TN0qVLRWtra3Hz5s3i2bNnxcmTJ9faNjg0NFQ8duyYeOjQIdHPz49tpTuwWbNmiW5ublI78g0bNoj29vbiiy++KF3Dz03nlpubK0ZGRoqRkZEiAPGTTz4RIyMjxbi4OFEUm+fzkZOTIzo5OYkPPfSQeP78eXHt2rWiubk525F3Nl988YXo6ekpmpiYiP379xePHj2q75JIDwDU+rVq1SrpmsLCQnHevHmijY2NaG5uLk6dOlVMTk7Wuc+NGzfECRMmiGZmZqK9vb343HPPiaWlpa38bkhfqgcnfmaour///lsMCgoSZTKZGBAQIH7zzTc65zUajfj666+LTk5OokwmE0eNGiXGxMToXJOZmSlOnz5dtLS0FOVyuThnzhwxNze3Nd8GtSKVSiUuXLhQ9PT0FE1NTcWuXbuKr776qk5baH5uOre9e/fW+jvMrFmzRFFsvs/HmTNnxCFDhogymUx0c3MTly5d2lpvUYcgilW2fyYiIiIiIqIauMaJiIiIiIioAQxOREREREREDWBwIiIiIiIiagCDExERERERUQMYnIiIiIiIiBrA4ERERERERNQABiciIiIiIqIGMDgRERERERE1gMGJiIioCQRBwKZNm/RdBhERtTIGJyIiajdmz54NQRBqfI0fP17fpRERUQdnpO8CiIiImmL8+PFYtWqVzjGZTKanaoiIqLPgiBMREbUrMpkMzs7OOl82NjYAtNPoli9fjgkTJsDMzAxdu3bF+vXrdZ5/7tw53HbbbTAzM4OdnR0ef/xx5OXl6Vzzww8/oEePHpDJZHBxccGCBQt0zmdkZGDq1KkwNzeHn58f/vrrr5Z900REpHcMTkRE1KG8/vrruOuuu3DmzBnMmDED999/Py5evAgAyM/Px7hx42BjY4MTJ05g3bp12LVrl04wWr58OebPn4/HH38c586dw19//QVfX1+d13j77bdx77334uzZs5g4cSJmzJiBrKysVn2fRETUugRRFEV9F0FERNQYs2fPxi+//AJTU1Od46+88gpeeeUVCIKAuXPnYvny5dK5gQMHonfv3vj666/x7bff4qWXXkJCQgIsLCwAAFu3bsUdd9yBpKQkODk5wc3NDXPmzMG7775baw2CIOC1117DkiVLAGjDmKWlJbZt28a1VkREHRjXOBERUbsycuRInWAEALa2ttL3YWFhOufCwsIQFRUFALh48SJCQkKk0AQAgwcPhkajQUxMDARBQFJSEkaNGlVvDcHBwdL3FhYWkMvlSEtLu9W3RERE7QCDExERtSsWFhY1ps41FzMzs0ZdZ2xsrPNYEARoNJqWKImIiNoIrnEiIqIO5ejRozUeBwYGAgACAwNx5swZ5OfnS+cPHz4MAwMD+Pv7w8rKCl5eXti9e3er1kxERG0fR5yIiKhdKS4uRkpKis4xIyMj2NvbAwDWrVuHvn37YsiQIfj1119x/PhxfP/99wCAGTNm4M0338SsWbPw1ltvIT09HU899RQeeughODk5AQDeeustzJ07F46OjpgwYQJyc3Nx+PBhPPXUU637RomIqE1hcCIionZl+/btcHFx0Tnm7++PS5cuAdB2vFu7di3mzZsHFxcX/Pbbb+jevTsAwNzcHDt27MDChQvRr18/mJub46677sInn3wi3WvWrFkoKirCp59+iueffx729va4++67W+8NEhFRm8SuekRE1GEIgoCNGzdiypQp+i6FiIg6GK5xIiIiIiIiagCDExERERERUQO4xomIiDoMzj4nIqKWwhEnIiIiIiKiBjA4ERERERERNYDBiYiIiIiIqAEMTkRERERERA1gcCIiIiIiImoAgxMREREREVEDGJyIiIiIiIgawOBERERERETUgP8DuDSzaar4pOcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "SAMPLE:\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/lv/fs2j5vr90zz71f7mbvytr3s80000gn/T/ipykernel_16440/2974536355.py:263: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  model.load_state_dict(torch.load('best_model.pt'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First off the rap in the way me hoby's pland open from here in moneys, gin'\n",
      "I'm from crise in the right Califorie, how tream but to recoh\n",
      "That's motherfuckers to party owna don't wanna find you brightal, they leam\n",
      "Suffer\n",
      "Like gone insan as a ride and my hishin' money is die\n",
      "Tripped, with me, Ou killingin' for me\n",
      "And life good to deal motherfuckin' at make, 0cause all you toper when I was shit I slow as a thug I got a kirs\n",
      "To the scoppin' (Live the life how I'm this is to a 15-shems by me will clock and with me\n",
      "And you wanmand it\n",
      "And a pool, but it's time you deated and back to the kight\n",
      "My motherfucker\n",
      "We was trive where when you, my little hoes\n",
      "Hown\n",
      "Comin' pocket rewn\n",
      "And a Bearly gettin' and these say the peace\n",
      "We've when I gift\n",
      "\n",
      "Soures, and didn't get you kill you gunna start to the streets?\n",
      "Yound how then it's on the West, it do from a here now?\n",
      "Thank 'cause it's crash wryin' to kepreciateds\n",
      "\" with shit I don't take through it\n",
      "And bulla stand mother be these police be lide's little all the l\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import string\n",
    "import random\n",
    "import time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "EPOCHS = 1000\n",
    "PRINT_EVERY = 1\n",
    "LEARNING_RATE = 1e-3\n",
    "PREDICT_LENGTH = 50 \n",
    "TEMPERATURE = 0.8\n",
    "CHUNK_LENGTH = 200\n",
    "CHARACTERS = string.printable\n",
    "INPUT_PATH = 'data/tupac.txt'\n",
    "INPUT_SIZE = OUTPUT_SIZE = len(CHARACTERS)\n",
    "HIDDEN_SIZE = 256\n",
    "FACTOR_SIZE = 256\n",
    "NUM_LAYERS = 1\n",
    "GRAD_CLIP = 0.25\n",
    "DROPOUT = 0.0\n",
    "WARMUP_EPOCHS = 100\n",
    "MIN_LR = 1e-5\n",
    "BATCH_SIZE = 64 \n",
    "WEIGHT_DECAY = 0.01\n",
    "\n",
    "\n",
    "class LayerNorm(nn.Module):\n",
    "    def __init__(self, normalized_shape, eps=1e-5):\n",
    "        super().__init__()\n",
    "        self.gamma = nn.Parameter(torch.ones(normalized_shape))\n",
    "        self.beta = nn.Parameter(torch.zeros(normalized_shape))\n",
    "        self.eps = eps\n",
    "        \n",
    "    def forward(self, x):\n",
    "        mean = x.mean(-1, keepdim=True)\n",
    "        var = x.var(-1, keepdim=True, unbiased=False)\n",
    "        return self.gamma * (x - mean) / (var + self.eps).sqrt() + self.beta\n",
    "\n",
    "class MultiplicativeRNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, factor_size, output_size, num_layers, dropout=0.1):\n",
    "        super().__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.factor_size = factor_size\n",
    "        self.num_layers = num_layers\n",
    "        self.dropout = dropout\n",
    "        \n",
    "        init_scale = 0.05\n",
    "        \n",
    "        self.encoder = nn.Embedding(input_size, hidden_size)\n",
    "        self.encoder_norm = LayerNorm(hidden_size)\n",
    "        \n",
    "        self.layers = nn.ModuleList([\n",
    "            nn.ModuleDict({\n",
    "                'W_fx': nn.Linear(hidden_size, factor_size, bias=False),\n",
    "                'W_fh': nn.Linear(hidden_size, factor_size, bias=False),\n",
    "                'W_hf': nn.Linear(factor_size, hidden_size, bias=True),\n",
    "                'layer_norm': LayerNorm(hidden_size)\n",
    "            }) for _ in range(num_layers)\n",
    "        ])\n",
    "        \n",
    "        self.decoder = nn.Linear(hidden_size, output_size)\n",
    "        \n",
    "        for name, p in self.named_parameters():\n",
    "            if p.dim() > 1:\n",
    "                if 'W_fx' in name or 'W_fh' in name:\n",
    "                    nn.init.orthogonal_(p, gain=init_scale)\n",
    "                else:\n",
    "                    nn.init.xavier_normal_(p, gain=init_scale)\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        batch_size = input.size(0)\n",
    "        encoded = self.encoder(input)\n",
    "        encoded = self.encoder_norm(encoded)\n",
    "        encoded = nn.functional.dropout(encoded, p=self.dropout, training=self.training)\n",
    "        \n",
    "        new_hidden = []\n",
    "        current_input = encoded\n",
    "        \n",
    "        for layer_idx, layer in enumerate(self.layers):\n",
    "            prev_hidden = hidden[layer_idx]\n",
    "            \n",
    "            f_t = layer['W_fx'](current_input) * torch.sigmoid(layer['W_fh'](prev_hidden))\n",
    "            h_t = torch.tanh(layer['W_hf'](f_t))\n",
    "            \n",
    "            h_t = layer['layer_norm'](h_t + current_input)\n",
    "            h_t = nn.functional.dropout(h_t, p=self.dropout, training=self.training)\n",
    "            \n",
    "            new_hidden.append(h_t)\n",
    "            current_input = h_t\n",
    "        \n",
    "        output = self.decoder(current_input)\n",
    "        hidden = torch.stack(new_hidden)\n",
    "        \n",
    "        return output, hidden\n",
    "\n",
    "    def init_hidden(self, batch_size):\n",
    "        return Variable(torch.zeros(self.num_layers, batch_size, self.hidden_size))\n",
    "\n",
    "def get_lr(epoch):\n",
    "    if epoch < WARMUP_EPOCHS:\n",
    "        return LEARNING_RATE * (epoch / WARMUP_EPOCHS)\n",
    "    progress = (epoch - WARMUP_EPOCHS) / (EPOCHS - WARMUP_EPOCHS)\n",
    "    return MIN_LR + (LEARNING_RATE - MIN_LR) * 0.5 * (1 + np.cos(np.pi * progress))\n",
    "\n",
    "def estimate_flops(model, sequence_length, batch_size):\n",
    "    H = HIDDEN_SIZE\n",
    "    F = FACTOR_SIZE\n",
    "    V = INPUT_SIZE\n",
    "    L = sequence_length\n",
    "    B = batch_size\n",
    "    \n",
    "    flops = B * L * (\n",
    "        2 * H * F +\n",
    "        2 * H * F +\n",
    "        F +\n",
    "        2 * F * H +\n",
    "        2 * H * V\n",
    "    )\n",
    "    return flops\n",
    "\n",
    "def char_tensor(string, target_length=None):\n",
    "    tensor = torch.zeros(len(string) if target_length is None else target_length).long()\n",
    "    for c in range(min(len(string), len(tensor))):\n",
    "        try:\n",
    "            tensor[c] = CHARACTERS.index(string[c])\n",
    "        except ValueError:\n",
    "            continue\n",
    "    return Variable(tensor)\n",
    "\n",
    "def get_batch(data, batch_size):\n",
    "    inputs = torch.zeros(batch_size, CHUNK_LENGTH).long()\n",
    "    targets = torch.zeros(batch_size, CHUNK_LENGTH).long()\n",
    "    \n",
    "    for b in range(batch_size):\n",
    "        start_idx = random.randint(0, len(data) - CHUNK_LENGTH - 1)\n",
    "        chunk = data[start_idx:start_idx + CHUNK_LENGTH + 1]\n",
    "        inputs[b] = char_tensor(chunk[:-1], CHUNK_LENGTH)\n",
    "        targets[b] = char_tensor(chunk[1:], CHUNK_LENGTH)\n",
    "    \n",
    "    return Variable(inputs), Variable(targets)\n",
    "\n",
    "def generate(model, prime_str='A', predict_len=100, temperature=0.8):\n",
    "    hidden = model.init_hidden(batch_size=1)\n",
    "    prime_input = char_tensor(prime_str)\n",
    "    predicted = prime_str\n",
    "\n",
    "    for p in range(len(prime_str)):\n",
    "        _, hidden = model(prime_input[p].unsqueeze(0), hidden)\n",
    "\n",
    "    inp = prime_input[-1]\n",
    "\n",
    "    for p in range(predict_len):\n",
    "        output, hidden = model(inp.unsqueeze(0), hidden)\n",
    "        output_dist = output.data.view(-1).div(temperature).exp()\n",
    "        top_idx = torch.multinomial(output_dist, 1)[0]\n",
    "        predicted_char = CHARACTERS[top_idx]\n",
    "        predicted += predicted_char\n",
    "        inp = char_tensor(predicted_char)\n",
    "\n",
    "    return predicted\n",
    "\n",
    "def print_model_info(model, data):\n",
    "    num_params = sum(p.numel() for p in model.parameters())\n",
    "    print(f\"\\nDATA\")\n",
    "    print(f\"Total characters: {len(data):,}\")\n",
    "    print(f\"Unique characters: {len(CHARACTERS)}\")\n",
    "    print(f\"Training chunk length: {CHUNK_LENGTH}\")\n",
    "    print(f\"Batch size: {BATCH_SIZE}\")\n",
    "    print(f\"\\nMODEL\")\n",
    "    print(f\"Input size: {INPUT_SIZE}\")\n",
    "    print(f\"Hidden size: {HIDDEN_SIZE}\")\n",
    "    print(f\"Factor size: {FACTOR_SIZE}\")\n",
    "    print(f\"Number of layers: {NUM_LAYERS}\")\n",
    "    print(f\"Total parameters: {num_params:,}\")\n",
    "    print(f\"\\nTRAIN\")\n",
    "    print(f\"Learning rate: {LEARNING_RATE}\")\n",
    "    print(f\"Number of epochs: {EPOCHS}\")\n",
    "    print(f\"Gradient clipping: {GRAD_CLIP}\")\n",
    "    print(f\"Weight decay: {WEIGHT_DECAY}\")\n",
    "    print(f\"Device: {'cuda' if torch.cuda.is_available() else 'cpu'}\")\n",
    "    print(\"-\" * 50 + \"\\n\")\n",
    "\n",
    "def train(model, data):\n",
    "    optimizer = torch.optim.AdamW(\n",
    "        model.parameters(),\n",
    "        lr=LEARNING_RATE,\n",
    "        weight_decay=WEIGHT_DECAY,\n",
    "        betas=(0.9, 0.95)\n",
    "    )\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    best_loss = float('inf')\n",
    "    losses = []\n",
    "    \n",
    "    try:\n",
    "        for epoch in range(EPOCHS):\n",
    "            model.train()\n",
    "            start_time = time.time()\n",
    "            \n",
    "            current_lr = get_lr(epoch)\n",
    "            for param_group in optimizer.param_groups:\n",
    "                param_group['lr'] = current_lr\n",
    "            \n",
    "            inp, target = get_batch(data, BATCH_SIZE)\n",
    "            hidden = model.init_hidden(BATCH_SIZE)\n",
    "            model.zero_grad()\n",
    "            loss = 0\n",
    "\n",
    "            for c in range(CHUNK_LENGTH):\n",
    "                output, hidden = model(inp[:, c], hidden)\n",
    "                loss += criterion(output.view(BATCH_SIZE, -1), target[:, c])\n",
    "\n",
    "            if torch.isnan(loss):\n",
    "                print(f\"NaN loss detected at epoch {epoch+1}. Stopping training.\")\n",
    "                break\n",
    "\n",
    "            loss.backward()\n",
    "            torch.nn.utils.clip_grad_norm_(model.parameters(), GRAD_CLIP)\n",
    "            optimizer.step()\n",
    "\n",
    "            loss = loss.item() / CHUNK_LENGTH\n",
    "            losses.append(loss)\n",
    "            \n",
    "            if loss < best_loss:\n",
    "                best_loss = loss\n",
    "                torch.save(model.state_dict(), 'best_model.pt')\n",
    "            \n",
    "            if (epoch + 1) % PRINT_EVERY == 0:\n",
    "                time_per_epoch = time.time() - start_time\n",
    "                flops = estimate_flops(model, CHUNK_LENGTH, BATCH_SIZE)\n",
    "                flops_per_sec = flops / time_per_epoch if time_per_epoch > 0 else 0\n",
    "                print(f'epoch: {epoch+1}/{EPOCHS}, loss: {loss:.4f}, mfu: {flops_per_sec/1e9:.2f}%')\n",
    "\n",
    "        plt.figure(figsize=(10, 5))\n",
    "        plt.plot(losses)\n",
    "        plt.title('Training Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.show()\n",
    "        plt.close()\n",
    "        \n",
    "    except KeyboardInterrupt:\n",
    "        print('Training interrupted')\n",
    "\n",
    "with open(INPUT_PATH, 'r', encoding='utf-8') as f:\n",
    "    data = f.read()\n",
    "\n",
    "model = MultiplicativeRNN(\n",
    "    INPUT_SIZE,\n",
    "    HIDDEN_SIZE,\n",
    "    FACTOR_SIZE,\n",
    "    OUTPUT_SIZE,\n",
    "    NUM_LAYERS,\n",
    "    DROPOUT\n",
    ")\n",
    "\n",
    "print_model_info(model, data)\n",
    "train(model, data)\n",
    "\n",
    "print(\"\\nSAMPLE:\\n\")\n",
    "model.load_state_dict(torch.load('best_model.pt'))\n",
    "print(generate(model, 'First off ', 1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First off the comin' shake it, and his this in the left and my life as a last the way it was they bustin' in the tarned on with me, I'm tring it's throw the scromptand my motherfucker\n",
      "Keep me hollin'\n",
      "\n",
      "The both\n",
      "And busty, my breathe smics when I get lady\n",
      "All your drove motherfuckanges, we let up? (Dear chottle shake it is\n",
      "\n",
      "We ain't drave the life on tell and all to keep it love the hate\n",
      "They got to make it up\n",
      "\n",
      "We life on the concrete\n",
      "Me un, F0cause no women\n",
      "All eyes on me\n",
      "And it's that I ain't niggas to de\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/lv/fs2j5vr90zz71f7mbvytr3s80000gn/T/ipykernel_16440/4252026083.py:9: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  model.load_state_dict(torch.load('best_model.pt'))\n"
     ]
    }
   ],
   "source": [
    "model = MultiplicativeRNN(\n",
    "        INPUT_SIZE,\n",
    "        HIDDEN_SIZE,\n",
    "        FACTOR_SIZE,\n",
    "        OUTPUT_SIZE,\n",
    "        NUM_LAYERS,\n",
    "        DROPOUT\n",
    "    )\n",
    "model.load_state_dict(torch.load('best_model.pt'))\n",
    "print(generate(model, 'First off ', 500))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}