3- Lab6 MNIST Beginner to Advanced - Convolutional Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.3\n",
    "set_session(tf.Session(config=config))\n",
    "\n",
    "notebook_num = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_kaggle_csv(matrix, header,filename):\n",
    "    frame = pd.DataFrame(data = matrix,columns=header)\n",
    "    frame.to_csv(path_or_buf  = filename,index = False,sep =',')\n",
    "    return frame\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "def load_train_valid_test_data(train_file,test_file,validation_percentage):\n",
    "    train_data = np.genfromtxt(delimiter=',',fname=train_file,skip_header=True)\n",
    "    test_data = np.genfromtxt(delimiter=',',fname=test_file,skip_header=True)\n",
    "    train_y = train_data[:,0]\n",
    "    train_x = train_data[:,1:]\n",
    "    train_x = train_x.reshape((train_data.shape[0],28,28,1))\n",
    "    enc = preprocessing.OneHotEncoder()\n",
    "    enc.fit(train_y.reshape((train_y.shape[0],1)))\n",
    "    train_y = enc.transform(train_y.reshape((train_y.shape[0],1))).toarray()\n",
    "    train_x /= 255\n",
    "    test_x = test_data.reshape((test_data.shape[0],28,28,1))\n",
    "    test_x  = test_x/255\n",
    "    train_x = train_x.astype('float32')\n",
    "    test_x = test_x.astype('float32')\n",
    "    train_x, valid_x, train_y, valid_y = train_test_split(train_x, train_y, test_size=0.2, random_state=42)\n",
    "    \n",
    "    return train_x, valid_x, test_x, train_y, valid_y\n",
    "\n",
    "def generate_submission(model,data,filename,file_columns = None):\n",
    "    predictions = model.predict(data)\n",
    "    result = predictions.argmax(axis =1 )\n",
    "    imgs_count = data.shape[0]\n",
    "    result = result.reshape((imgs_count,1))\n",
    "    ids = (np.arange(imgs_count)+1).reshape((imgs_count,1))\n",
    "    result = np.hstack((ids,result))\n",
    "    m = to_kaggle_csv(result,file_columns, filename)\n",
    "\n",
    "def validate_preprocessing(train_x, valid_x, test_x, train_y, valid_y):\n",
    "    print('max train',train_x.max())\n",
    "    print('max valid',train_x.max())\n",
    "    print('max test',train_x.max())\n",
    "    \n",
    "    print('train_x shape',train_x.shape)\n",
    "    print('valid_x shape',valid_x.shape)\n",
    "    print('test_x shape',test_x.shape)\n",
    "    print('train_y shape',train_y.shape)\n",
    "    print('valid_y shape',valid_y.shape)\n",
    "    \n",
    "def history_appending(history1,history2):\n",
    "    history1.history['acc'] += history2.history['acc']\n",
    "    history1.history['val_acc'] += history2.history['val_acc']\n",
    "    history1.history['val_loss'] += history2.history['val_loss']\n",
    "    history1.history['loss'] += history2.history['loss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, valid_x, test_x, train_y, valid_y = load_train_valid_test_data('train.csv','test.csv',0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max train 1.0\n",
      "max valid 1.0\n",
      "max test 1.0\n",
      "train_x shape (33600, 28, 28, 1)\n",
      "valid_x shape (8400, 28, 28, 1)\n",
      "test_x shape (28000, 28, 28, 1)\n",
      "train_y shape (33600, 10)\n",
      "valid_y shape (8400, 10)\n"
     ]
    }
   ],
   "source": [
    "validate_preprocessing(train_x, valid_x, test_x, train_y, valid_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training a convolutional neural network for mnist dataset\n",
    "multilayer networks have the capacity to learn features of the input and build upon it in subsequent layers, so it can overfit to the training data, we'll need to make sure it doesn't, that's why we'll be monitoring the training process through both validation and training accuracies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential,Model\n",
    "from keras import optimizers\n",
    "from keras import regularizers\n",
    "from keras.layers.core import Dense, Activation# defining the layers\n",
    "from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Dropout, Flatten\n",
    "from keras.optimizers import RMSprop, SGD , Adam\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_conv(model,filters):\n",
    "    model.add(Conv2D(filters,(3,3),padding='same'))\n",
    "    model.add(Activation('relu'))\n",
    "    return model\n",
    "    \n",
    "\n",
    "def CNN_architecture(Batch_Norm,dropout):\n",
    "    cnn_model = Sequential()    \n",
    "    cnn_model.add(Conv2D(32,(3,3),padding='same',input_shape=(28,28,1)))\n",
    "    cnn_model.add(Activation('relu'))\n",
    "    cnn_model = add_conv(cnn_model,32)\n",
    "    cnn_model.add(MaxPooling2D(pool_size =(2,2),strides = (2,2)))\n",
    "    cnn_model.add(Dropout(0.2))\n",
    "    cnn_model = add_conv(cnn_model,64)\n",
    "    cnn_model = add_conv(cnn_model,64)\n",
    "    cnn_model.add(MaxPooling2D(pool_size = (2,2),strides = (2,2)))\n",
    "    cnn_model.add(Dropout(0.2))\n",
    "    cnn_model.add(Flatten())\n",
    "    cnn_model.add(Dense(64))\n",
    "    cnn_model.add(Activation('relu'))\n",
    "    if Batch_Norm:\n",
    "        cnn_model.add(BatchNormalization())\n",
    "    cnn_model.add(Dropout(drop_out))\n",
    "    cnn_model.add(Dense(32))\n",
    "    cnn_model.add(Activation('relu'))\n",
    "    if Batch_Norm:\n",
    "        cnn_model.add(BatchNormalization())\n",
    "    cnn_model.add(Dropout(drop_out))\n",
    "    cnn_model.add(Dense(10))\n",
    "    cnn_model.add(Activation('softmax'))\n",
    "    return cnn_model\n",
    "\n",
    "def CNN_model_1(optimizer,epochs,train_x,train_y,valid_x = None,valid_y = None, Batch_Norm = False,drop_out = 0.5):\n",
    "    cnn_model = CNN_architecture(Batch_Norm,drop_out)\n",
    "    optimizer = Adam(lr= 0.001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "    history = cnn_model.fit(train_x,train_y,\n",
    "                        batch_size = 64,epochs=int(epochs),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "    return cnn_model,history\n",
    "\n",
    "def CNN_model_2(optimizer,epochs,train_x,train_y,valid_x = None,valid_y = None, Batch_Norm = False,drop_out = 0.5):\n",
    "    cnn_model = CNN_architecture(Batch_Norm,drop_out)\n",
    "    optimizer = Adam(lr= 0.001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizer,#RMSprop()\n",
    "                  metrics=['accuracy'])\n",
    "    history = cnn_model.fit(train_x,train_y,\n",
    "                        batch_size = 64,epochs=int(epochs*2/3),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "    optimizer = Adam(lr= 0.0001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizer,#RMSprop()\n",
    "                  metrics=['accuracy'])\n",
    "    history = cnn_model.fit(train_x,train_y,\n",
    "                        batch_size = 128,epochs=int(epochs*1/3),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "    optimizer = Adam(lr= 0.00001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizer,#RMSprop()\n",
    "                  metrics=['accuracy'])\n",
    "    history = cnn_model.fit(train_x,train_y,\n",
    "                        batch_size = 128,epochs=int(epochs*1/3),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "    \n",
    "    return cnn_model,history\n",
    "\n",
    "def CNN_model_3(optimizer,epochs,train_x,train_y,valid_x = None,valid_y = None, Batch_Norm = False,drop_out = 0.5):\n",
    "    cnn_model = CNN_architecture(Batch_Norm,drop_out)\n",
    "    for i in range(3):\n",
    "        optimizer = Adam(lr= 0.001)\n",
    "        cnn_model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=optimizer,#RMSprop()\n",
    "                      metrics=['accuracy'])\n",
    "        history = cnn_model.fit(train_x,train_y,\n",
    "                            batch_size = 64,epochs=int(epochs*2/3),\n",
    "                            verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "        optimizer = Adam(lr= 0.0001)\n",
    "        cnn_model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=optimizer,#RMSprop()\n",
    "                      metrics=['accuracy'])\n",
    "        history = cnn_model.fit(train_x,train_y,\n",
    "                            batch_size = 128,epochs=int(epochs*2/3),\n",
    "                            verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "    return cnn_model,history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Normalization:  True\n",
      "Technique  learning rate decay and bs increase\n",
      "drop_out  0.35\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'train_x' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-f83c8d27b641>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCNN_model_1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_y\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdrop_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrop_out\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mBatch_Norm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mtech\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m                 \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mhistory\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0mCNN_model_2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_y\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdrop_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrop_out\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mBatch_Norm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mtech\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mhistory\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0mCNN_model_3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_y\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdrop_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrop_out\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mBatch_Norm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'train_x' is not defined"
     ]
    }
   ],
   "source": [
    "drop_outs = [0.35,0.4,0.45]\n",
    "techniques = [1,2]\n",
    "models = []\n",
    "batch_enum = [True]\n",
    "histories = []\n",
    "technique_dict = {0:\"directly training\",1:\"learning rate decay and bs increase\",2: \"learning rate schedule restarting\"}\n",
    "optimizer = 'adam'\n",
    "n_epochs = 40\n",
    "for drop_out in drop_outs:\n",
    "    for tech in techniques:\n",
    "        for batch in batch_enum:\n",
    "            print(\"Batch Normalization: \", batch)\n",
    "            print(\"Technique \", technique_dict[tech])\n",
    "            print(\"drop_out \", drop_out)\n",
    "            if tech == 0:\n",
    "                model,history = CNN_model_1(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out,Batch_Norm=batch)\n",
    "            elif tech == 1:\n",
    "                model,history  = CNN_model_2(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out,Batch_Norm=batch)\n",
    "            elif tech == 2:\n",
    "                model,history  = CNN_model_3(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out,Batch_Norm=batch)\n",
    "            models.append(model)\n",
    "            histories.append(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Normalization:  True\n",
      "Technique  learning rate schedule restarting\n",
      "drop_out  0.5\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/26\n",
      "33600/33600 [==============================] - 20s 594us/step - loss: 0.7471 - acc: 0.7716 - val_loss: 0.1090 - val_acc: 0.9765\n",
      "Epoch 2/26\n",
      "33600/33600 [==============================] - 11s 334us/step - loss: 0.2472 - acc: 0.9316 - val_loss: 0.0561 - val_acc: 0.9819\n",
      "Epoch 3/26\n",
      "33600/33600 [==============================] - 11s 335us/step - loss: 0.1605 - acc: 0.9569 - val_loss: 0.0478 - val_acc: 0.9858\n",
      "Epoch 4/26\n",
      "33600/33600 [==============================] - 11s 342us/step - loss: 0.1286 - acc: 0.9661 - val_loss: 0.0348 - val_acc: 0.9888\n",
      "Epoch 5/26\n",
      "33600/33600 [==============================] - 11s 334us/step - loss: 0.1044 - acc: 0.9717 - val_loss: 0.0433 - val_acc: 0.9870\n",
      "Epoch 6/26\n",
      "33600/33600 [==============================] - 11s 336us/step - loss: 0.0970 - acc: 0.9740 - val_loss: 0.0321 - val_acc: 0.9896\n",
      "Epoch 7/26\n",
      "33600/33600 [==============================] - 11s 333us/step - loss: 0.0835 - acc: 0.9776 - val_loss: 0.0270 - val_acc: 0.9908\n",
      "Epoch 8/26\n",
      "33600/33600 [==============================] - 11s 331us/step - loss: 0.0789 - acc: 0.9790 - val_loss: 0.0317 - val_acc: 0.9910\n",
      "Epoch 9/26\n",
      "33600/33600 [==============================] - 11s 337us/step - loss: 0.0735 - acc: 0.9807 - val_loss: 0.0273 - val_acc: 0.9913\n",
      "Epoch 10/26\n",
      "33600/33600 [==============================] - 11s 335us/step - loss: 0.0666 - acc: 0.9834 - val_loss: 0.0275 - val_acc: 0.9914\n",
      "Epoch 11/26\n",
      "33600/33600 [==============================] - 11s 332us/step - loss: 0.0618 - acc: 0.9834 - val_loss: 0.0271 - val_acc: 0.9915\n",
      "Epoch 12/26\n",
      "33600/33600 [==============================] - 11s 337us/step - loss: 0.0578 - acc: 0.9846 - val_loss: 0.0311 - val_acc: 0.9912\n",
      "Epoch 13/26\n",
      "33600/33600 [==============================] - 11s 333us/step - loss: 0.0570 - acc: 0.9848 - val_loss: 0.0266 - val_acc: 0.9931\n",
      "Epoch 14/26\n",
      "33600/33600 [==============================] - 11s 338us/step - loss: 0.0535 - acc: 0.9861 - val_loss: 0.0255 - val_acc: 0.9925\n",
      "Epoch 15/26\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0506 - acc: 0.9867 - val_loss: 0.0269 - val_acc: 0.9924\n",
      "Epoch 16/26\n",
      "33600/33600 [==============================] - 11s 335us/step - loss: 0.0516 - acc: 0.9863 - val_loss: 0.0259 - val_acc: 0.9936\n",
      "Epoch 17/26\n",
      "33600/33600 [==============================] - 11s 335us/step - loss: 0.0473 - acc: 0.9875 - val_loss: 0.0217 - val_acc: 0.9937\n",
      "Epoch 18/26\n",
      "33600/33600 [==============================] - 11s 333us/step - loss: 0.0463 - acc: 0.9874 - val_loss: 0.0261 - val_acc: 0.9921\n",
      "Epoch 19/26\n",
      "33600/33600 [==============================] - 11s 333us/step - loss: 0.0437 - acc: 0.9889 - val_loss: 0.0260 - val_acc: 0.9929\n",
      "Epoch 20/26\n",
      "33600/33600 [==============================] - 11s 336us/step - loss: 0.0425 - acc: 0.9882 - val_loss: 0.0271 - val_acc: 0.9929\n",
      "Epoch 21/26\n",
      "33600/33600 [==============================] - 11s 333us/step - loss: 0.0422 - acc: 0.9890 - val_loss: 0.0230 - val_acc: 0.9937\n",
      "Epoch 22/26\n",
      "33600/33600 [==============================] - 11s 332us/step - loss: 0.0367 - acc: 0.9899 - val_loss: 0.0186 - val_acc: 0.9943\n",
      "Epoch 23/26\n",
      "33600/33600 [==============================] - 11s 334us/step - loss: 0.0361 - acc: 0.9899 - val_loss: 0.0283 - val_acc: 0.9929\n",
      "Epoch 24/26\n",
      "33600/33600 [==============================] - 11s 335us/step - loss: 0.0355 - acc: 0.9903 - val_loss: 0.0256 - val_acc: 0.9942\n",
      "Epoch 25/26\n",
      "33600/33600 [==============================] - 11s 336us/step - loss: 0.0373 - acc: 0.9900 - val_loss: 0.0229 - val_acc: 0.9942\n",
      "Epoch 26/26\n",
      "33600/33600 [==============================] - 11s 334us/step - loss: 0.0317 - acc: 0.9908 - val_loss: 0.0270 - val_acc: 0.9940\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/26\n",
      "33600/33600 [==============================] - 17s 505us/step - loss: 0.0220 - acc: 0.9947 - val_loss: 0.0205 - val_acc: 0.9958\n",
      "Epoch 2/26\n",
      "33600/33600 [==============================] - 8s 242us/step - loss: 0.0203 - acc: 0.9951 - val_loss: 0.0209 - val_acc: 0.9956\n",
      "Epoch 3/26\n",
      "33600/33600 [==============================] - 8s 243us/step - loss: 0.0188 - acc: 0.9949 - val_loss: 0.0203 - val_acc: 0.9960\n",
      "Epoch 4/26\n",
      "33600/33600 [==============================] - 8s 244us/step - loss: 0.0184 - acc: 0.9948 - val_loss: 0.0198 - val_acc: 0.9955\n",
      "Epoch 5/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0170 - acc: 0.9956 - val_loss: 0.0193 - val_acc: 0.9960\n",
      "Epoch 6/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0146 - acc: 0.9961 - val_loss: 0.0194 - val_acc: 0.9960\n",
      "Epoch 7/26\n",
      "33600/33600 [==============================] - 8s 244us/step - loss: 0.0156 - acc: 0.9958 - val_loss: 0.0199 - val_acc: 0.9957\n",
      "Epoch 8/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0169 - acc: 0.9956 - val_loss: 0.0191 - val_acc: 0.9960\n",
      "Epoch 9/26\n",
      "33600/33600 [==============================] - 8s 241us/step - loss: 0.0141 - acc: 0.9960 - val_loss: 0.0186 - val_acc: 0.9958\n",
      "Epoch 10/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0146 - acc: 0.9959 - val_loss: 0.0188 - val_acc: 0.9961\n",
      "Epoch 11/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0135 - acc: 0.9966 - val_loss: 0.0191 - val_acc: 0.9963\n",
      "Epoch 12/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0127 - acc: 0.9965 - val_loss: 0.0190 - val_acc: 0.9962\n",
      "Epoch 13/26\n",
      "33600/33600 [==============================] - 8s 242us/step - loss: 0.0141 - acc: 0.9963 - val_loss: 0.0190 - val_acc: 0.9962\n",
      "Epoch 14/26\n",
      "33600/33600 [==============================] - 8s 243us/step - loss: 0.0131 - acc: 0.9966 - val_loss: 0.0191 - val_acc: 0.9962\n",
      "Epoch 15/26\n",
      "33600/33600 [==============================] - 8s 242us/step - loss: 0.0140 - acc: 0.9965 - val_loss: 0.0185 - val_acc: 0.9962\n",
      "Epoch 16/26\n",
      "33600/33600 [==============================] - 8s 243us/step - loss: 0.0131 - acc: 0.9966 - val_loss: 0.0192 - val_acc: 0.9957\n",
      "Epoch 17/26\n",
      "33600/33600 [==============================] - 8s 244us/step - loss: 0.0127 - acc: 0.9968 - val_loss: 0.0191 - val_acc: 0.9957\n",
      "Epoch 18/26\n",
      "33600/33600 [==============================] - 8s 243us/step - loss: 0.0121 - acc: 0.9968 - val_loss: 0.0195 - val_acc: 0.9957\n",
      "Epoch 19/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0125 - acc: 0.9966 - val_loss: 0.0199 - val_acc: 0.9958\n",
      "Epoch 20/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0111 - acc: 0.9975 - val_loss: 0.0196 - val_acc: 0.9958\n",
      "Epoch 21/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0135 - acc: 0.9963 - val_loss: 0.0191 - val_acc: 0.9957\n",
      "Epoch 22/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0112 - acc: 0.9968 - val_loss: 0.0188 - val_acc: 0.9960\n",
      "Epoch 23/26\n",
      "33600/33600 [==============================] - 8s 243us/step - loss: 0.0108 - acc: 0.9970 - val_loss: 0.0191 - val_acc: 0.9960\n",
      "Epoch 24/26\n",
      "33600/33600 [==============================] - 8s 243us/step - loss: 0.0108 - acc: 0.9971 - val_loss: 0.0193 - val_acc: 0.9960\n",
      "Epoch 25/26\n",
      "33600/33600 [==============================] - 8s 244us/step - loss: 0.0116 - acc: 0.9972 - val_loss: 0.0195 - val_acc: 0.9958\n",
      "Epoch 26/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0126 - acc: 0.9968 - val_loss: 0.0196 - val_acc: 0.9960\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/26\n",
      "33600/33600 [==============================] - 20s 608us/step - loss: 0.0281 - acc: 0.9926 - val_loss: 0.0554 - val_acc: 0.9900\n",
      "Epoch 2/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0335 - acc: 0.9907 - val_loss: 0.0203 - val_acc: 0.9948\n",
      "Epoch 3/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0276 - acc: 0.9923 - val_loss: 0.0217 - val_acc: 0.9943\n",
      "Epoch 4/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0254 - acc: 0.9926 - val_loss: 0.0228 - val_acc: 0.9943\n",
      "Epoch 5/26\n",
      "33600/33600 [==============================] - 12s 343us/step - loss: 0.0256 - acc: 0.9921 - val_loss: 0.0232 - val_acc: 0.9950\n",
      "Epoch 6/26\n",
      "33600/33600 [==============================] - 12s 343us/step - loss: 0.0294 - acc: 0.9921 - val_loss: 0.0284 - val_acc: 0.9936\n",
      "Epoch 7/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0304 - acc: 0.9922 - val_loss: 0.0247 - val_acc: 0.9939\n",
      "Epoch 8/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0246 - acc: 0.9927 - val_loss: 0.0265 - val_acc: 0.9936\n",
      "Epoch 9/26\n",
      "33600/33600 [==============================] - 11s 342us/step - loss: 0.0234 - acc: 0.9937 - val_loss: 0.0260 - val_acc: 0.9943\n",
      "Epoch 10/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0254 - acc: 0.9926 - val_loss: 0.0269 - val_acc: 0.9944\n",
      "Epoch 11/26\n",
      "33600/33600 [==============================] - 11s 338us/step - loss: 0.0232 - acc: 0.9931 - val_loss: 0.0265 - val_acc: 0.9940\n",
      "Epoch 12/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0242 - acc: 0.9935 - val_loss: 0.0284 - val_acc: 0.9942\n",
      "Epoch 13/26\n",
      "33600/33600 [==============================] - 11s 342us/step - loss: 0.0263 - acc: 0.9935 - val_loss: 0.0267 - val_acc: 0.9937\n",
      "Epoch 14/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0206 - acc: 0.9938 - val_loss: 0.0299 - val_acc: 0.9938\n",
      "Epoch 15/26\n",
      "33600/33600 [==============================] - 11s 338us/step - loss: 0.0236 - acc: 0.9934 - val_loss: 0.0260 - val_acc: 0.9948\n",
      "Epoch 16/26\n",
      "33600/33600 [==============================] - 12s 345us/step - loss: 0.0203 - acc: 0.9939 - val_loss: 0.0276 - val_acc: 0.9944\n",
      "Epoch 17/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0222 - acc: 0.9940 - val_loss: 0.0296 - val_acc: 0.9948\n",
      "Epoch 18/26\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0201 - acc: 0.9947 - val_loss: 0.0293 - val_acc: 0.9933\n",
      "Epoch 19/26\n",
      "33600/33600 [==============================] - 11s 338us/step - loss: 0.0201 - acc: 0.9939 - val_loss: 0.0295 - val_acc: 0.9944\n",
      "Epoch 20/26\n",
      "33600/33600 [==============================] - 11s 336us/step - loss: 0.0209 - acc: 0.9940 - val_loss: 0.0264 - val_acc: 0.9946\n",
      "Epoch 21/26\n",
      "33600/33600 [==============================] - 12s 342us/step - loss: 0.0212 - acc: 0.9940 - val_loss: 0.0275 - val_acc: 0.9943\n",
      "Epoch 22/26\n",
      "33600/33600 [==============================] - 12s 343us/step - loss: 0.0186 - acc: 0.9949 - val_loss: 0.0232 - val_acc: 0.9957\n",
      "Epoch 23/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0160 - acc: 0.9955 - val_loss: 0.0238 - val_acc: 0.9951\n",
      "Epoch 24/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0185 - acc: 0.9951 - val_loss: 0.0329 - val_acc: 0.9939\n",
      "Epoch 25/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0221 - acc: 0.9941 - val_loss: 0.0299 - val_acc: 0.9942\n",
      "Epoch 26/26\n",
      "33600/33600 [==============================] - 11s 342us/step - loss: 0.0204 - acc: 0.9938 - val_loss: 0.0264 - val_acc: 0.9957\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/26\n",
      "33600/33600 [==============================] - 17s 519us/step - loss: 0.0128 - acc: 0.9969 - val_loss: 0.0236 - val_acc: 0.9956\n",
      "Epoch 2/26\n",
      "33600/33600 [==============================] - 8s 249us/step - loss: 0.0103 - acc: 0.9972 - val_loss: 0.0233 - val_acc: 0.9958\n",
      "Epoch 3/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0115 - acc: 0.9969 - val_loss: 0.0230 - val_acc: 0.9960\n",
      "Epoch 4/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0105 - acc: 0.9971 - val_loss: 0.0230 - val_acc: 0.9958\n",
      "Epoch 5/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0091 - acc: 0.9974 - val_loss: 0.0233 - val_acc: 0.9958\n",
      "Epoch 6/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0097 - acc: 0.9976 - val_loss: 0.0230 - val_acc: 0.9961\n",
      "Epoch 7/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0080 - acc: 0.9976 - val_loss: 0.0231 - val_acc: 0.9960\n",
      "Epoch 8/26\n",
      "33600/33600 [==============================] - 8s 250us/step - loss: 0.0084 - acc: 0.9977 - val_loss: 0.0228 - val_acc: 0.9957\n",
      "Epoch 9/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0074 - acc: 0.9979 - val_loss: 0.0228 - val_acc: 0.9960\n",
      "Epoch 10/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0078 - acc: 0.9977 - val_loss: 0.0236 - val_acc: 0.9955\n",
      "Epoch 11/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0089 - acc: 0.9974 - val_loss: 0.0235 - val_acc: 0.9955\n",
      "Epoch 12/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0076 - acc: 0.9978 - val_loss: 0.0241 - val_acc: 0.9956\n",
      "Epoch 13/26\n",
      "33600/33600 [==============================] - 8s 251us/step - loss: 0.0079 - acc: 0.9979 - val_loss: 0.0234 - val_acc: 0.9954\n",
      "Epoch 14/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0074 - acc: 0.9982 - val_loss: 0.0231 - val_acc: 0.9952\n",
      "Epoch 15/26\n",
      "33600/33600 [==============================] - 8s 249us/step - loss: 0.0077 - acc: 0.9981 - val_loss: 0.0233 - val_acc: 0.9952\n",
      "Epoch 16/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0072 - acc: 0.9979 - val_loss: 0.0224 - val_acc: 0.9956\n",
      "Epoch 17/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0074 - acc: 0.9981 - val_loss: 0.0225 - val_acc: 0.9957\n",
      "Epoch 18/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0062 - acc: 0.9981 - val_loss: 0.0229 - val_acc: 0.9958\n",
      "Epoch 19/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0055 - acc: 0.9987 - val_loss: 0.0231 - val_acc: 0.9955\n",
      "Epoch 20/26\n",
      "33600/33600 [==============================] - 8s 251us/step - loss: 0.0061 - acc: 0.9985 - val_loss: 0.0236 - val_acc: 0.9956\n",
      "Epoch 21/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0073 - acc: 0.9978 - val_loss: 0.0229 - val_acc: 0.9952\n",
      "Epoch 22/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0065 - acc: 0.9982 - val_loss: 0.0236 - val_acc: 0.9955\n",
      "Epoch 23/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0068 - acc: 0.9980 - val_loss: 0.0228 - val_acc: 0.9951\n",
      "Epoch 24/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0064 - acc: 0.9980 - val_loss: 0.0227 - val_acc: 0.9956\n",
      "Epoch 25/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0074 - acc: 0.9978 - val_loss: 0.0235 - val_acc: 0.9954\n",
      "Epoch 26/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0065 - acc: 0.9980 - val_loss: 0.0230 - val_acc: 0.9958\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/26\n",
      "33600/33600 [==============================] - 21s 628us/step - loss: 0.0140 - acc: 0.9963 - val_loss: 0.0304 - val_acc: 0.9940\n",
      "Epoch 2/26\n",
      "33600/33600 [==============================] - 12s 362us/step - loss: 0.0145 - acc: 0.9955 - val_loss: 0.0284 - val_acc: 0.9948\n",
      "Epoch 3/26\n",
      "33600/33600 [==============================] - 12s 350us/step - loss: 0.0178 - acc: 0.9948 - val_loss: 0.0261 - val_acc: 0.9950\n",
      "Epoch 4/26\n",
      "33600/33600 [==============================] - 12s 355us/step - loss: 0.0147 - acc: 0.9956 - val_loss: 0.0269 - val_acc: 0.9951\n",
      "Epoch 5/26\n",
      "33600/33600 [==============================] - 12s 353us/step - loss: 0.0168 - acc: 0.9953 - val_loss: 0.0280 - val_acc: 0.9946\n",
      "Epoch 6/26\n",
      "33600/33600 [==============================] - 12s 357us/step - loss: 0.0198 - acc: 0.9946 - val_loss: 0.0224 - val_acc: 0.9954\n",
      "Epoch 7/26\n",
      "33600/33600 [==============================] - 12s 362us/step - loss: 0.0172 - acc: 0.9951 - val_loss: 0.0268 - val_acc: 0.9946\n",
      "Epoch 8/26\n",
      "33600/33600 [==============================] - 12s 347us/step - loss: 0.0176 - acc: 0.9948 - val_loss: 0.0372 - val_acc: 0.9929\n",
      "Epoch 9/26\n",
      "33600/33600 [==============================] - 12s 368us/step - loss: 0.0191 - acc: 0.9943 - val_loss: 0.0308 - val_acc: 0.9940\n",
      "Epoch 10/26\n",
      "33600/33600 [==============================] - 13s 383us/step - loss: 0.0130 - acc: 0.9962 - val_loss: 0.0242 - val_acc: 0.9956\n",
      "Epoch 11/26\n",
      "33600/33600 [==============================] - 13s 393us/step - loss: 0.0138 - acc: 0.9959 - val_loss: 0.0276 - val_acc: 0.9951\n",
      "Epoch 12/26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33600/33600 [==============================] - 13s 398us/step - loss: 0.0151 - acc: 0.9955 - val_loss: 0.0263 - val_acc: 0.9951\n",
      "Epoch 13/26\n",
      "33600/33600 [==============================] - 13s 393us/step - loss: 0.0148 - acc: 0.9957 - val_loss: 0.0261 - val_acc: 0.9946\n",
      "Epoch 14/26\n",
      "33600/33600 [==============================] - 13s 387us/step - loss: 0.0160 - acc: 0.9953 - val_loss: 0.0258 - val_acc: 0.9949\n",
      "Epoch 15/26\n",
      "33600/33600 [==============================] - 12s 347us/step - loss: 0.0154 - acc: 0.9959 - val_loss: 0.0286 - val_acc: 0.9944\n",
      "Epoch 16/26\n",
      "33600/33600 [==============================] - 12s 343us/step - loss: 0.0174 - acc: 0.9952 - val_loss: 0.0243 - val_acc: 0.9955\n",
      "Epoch 17/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0152 - acc: 0.9959 - val_loss: 0.0262 - val_acc: 0.9943\n",
      "Epoch 18/26\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0154 - acc: 0.9958 - val_loss: 0.0281 - val_acc: 0.9946\n",
      "Epoch 19/26\n",
      "33600/33600 [==============================] - 12s 344us/step - loss: 0.0145 - acc: 0.9961 - val_loss: 0.0281 - val_acc: 0.9950\n",
      "Epoch 20/26\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0149 - acc: 0.9960 - val_loss: 0.0312 - val_acc: 0.9942\n",
      "Epoch 21/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0153 - acc: 0.9953 - val_loss: 0.0333 - val_acc: 0.9944\n",
      "Epoch 22/26\n",
      "33600/33600 [==============================] - 11s 340us/step - loss: 0.0146 - acc: 0.9959 - val_loss: 0.0335 - val_acc: 0.9943\n",
      "Epoch 23/26\n",
      "33600/33600 [==============================] - 11s 339us/step - loss: 0.0148 - acc: 0.9959 - val_loss: 0.0278 - val_acc: 0.9952\n",
      "Epoch 24/26\n",
      "33600/33600 [==============================] - 11s 337us/step - loss: 0.0141 - acc: 0.9962 - val_loss: 0.0258 - val_acc: 0.9950\n",
      "Epoch 25/26\n",
      "33600/33600 [==============================] - 11s 341us/step - loss: 0.0124 - acc: 0.9961 - val_loss: 0.0269 - val_acc: 0.9952\n",
      "Epoch 26/26\n",
      "33600/33600 [==============================] - 12s 342us/step - loss: 0.0124 - acc: 0.9964 - val_loss: 0.0313 - val_acc: 0.9951\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/26\n",
      "33600/33600 [==============================] - 18s 525us/step - loss: 0.0106 - acc: 0.9969 - val_loss: 0.0280 - val_acc: 0.9955\n",
      "Epoch 2/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0088 - acc: 0.9980 - val_loss: 0.0271 - val_acc: 0.9956\n",
      "Epoch 3/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0072 - acc: 0.9982 - val_loss: 0.0262 - val_acc: 0.9951\n",
      "Epoch 4/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0070 - acc: 0.9980 - val_loss: 0.0264 - val_acc: 0.9954\n",
      "Epoch 5/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0072 - acc: 0.9979 - val_loss: 0.0254 - val_acc: 0.9955\n",
      "Epoch 6/26\n",
      "33600/33600 [==============================] - 8s 249us/step - loss: 0.0072 - acc: 0.9981 - val_loss: 0.0259 - val_acc: 0.9955\n",
      "Epoch 7/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0070 - acc: 0.9982 - val_loss: 0.0253 - val_acc: 0.9956\n",
      "Epoch 8/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0064 - acc: 0.9980 - val_loss: 0.0250 - val_acc: 0.9954\n",
      "Epoch 9/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0065 - acc: 0.9980 - val_loss: 0.0253 - val_acc: 0.9957\n",
      "Epoch 10/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0066 - acc: 0.9984 - val_loss: 0.0256 - val_acc: 0.9956\n",
      "Epoch 11/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0056 - acc: 0.9985 - val_loss: 0.0255 - val_acc: 0.9954\n",
      "Epoch 12/26\n",
      "33600/33600 [==============================] - 8s 249us/step - loss: 0.0068 - acc: 0.9982 - val_loss: 0.0262 - val_acc: 0.9960\n",
      "Epoch 13/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0068 - acc: 0.9980 - val_loss: 0.0260 - val_acc: 0.9956\n",
      "Epoch 14/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0061 - acc: 0.9981 - val_loss: 0.0261 - val_acc: 0.9955\n",
      "Epoch 15/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0062 - acc: 0.9980 - val_loss: 0.0263 - val_acc: 0.9957\n",
      "Epoch 16/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0059 - acc: 0.9986 - val_loss: 0.0258 - val_acc: 0.9955\n",
      "Epoch 17/26\n",
      "33600/33600 [==============================] - 8s 245us/step - loss: 0.0059 - acc: 0.9982 - val_loss: 0.0249 - val_acc: 0.9956\n",
      "Epoch 18/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0063 - acc: 0.9984 - val_loss: 0.0250 - val_acc: 0.9955\n",
      "Epoch 19/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0050 - acc: 0.9986 - val_loss: 0.0246 - val_acc: 0.9955\n",
      "Epoch 20/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0051 - acc: 0.9986 - val_loss: 0.0246 - val_acc: 0.9957\n",
      "Epoch 21/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0056 - acc: 0.9986 - val_loss: 0.0245 - val_acc: 0.9958\n",
      "Epoch 22/26\n",
      "33600/33600 [==============================] - 8s 250us/step - loss: 0.0048 - acc: 0.9986 - val_loss: 0.0244 - val_acc: 0.9957\n",
      "Epoch 23/26\n",
      "33600/33600 [==============================] - 8s 246us/step - loss: 0.0042 - acc: 0.9987 - val_loss: 0.0246 - val_acc: 0.9960\n",
      "Epoch 24/26\n",
      "33600/33600 [==============================] - 8s 247us/step - loss: 0.0059 - acc: 0.9983 - val_loss: 0.0245 - val_acc: 0.9958\n",
      "Epoch 25/26\n",
      "33600/33600 [==============================] - 8s 248us/step - loss: 0.0048 - acc: 0.9987 - val_loss: 0.0245 - val_acc: 0.9958\n",
      "Epoch 26/26\n",
      "33600/33600 [==============================] - 8s 249us/step - loss: 0.0051 - acc: 0.9985 - val_loss: 0.0248 - val_acc: 0.9956\n"
     ]
    }
   ],
   "source": [
    "drop_outs = [0.50]\n",
    "techniques = [2]\n",
    "models = []\n",
    "batch_enum = [True]\n",
    "histories = []\n",
    "technique_dict = {0:\"directly training\",1:\"learning rate decay and bs increase\",2: \"learning rate schedule restarting\"}\n",
    "optimizer = 'adam'\n",
    "n_epochs = 40\n",
    "for drop_out in drop_outs:\n",
    "    for tech in techniques:\n",
    "        for batch in batch_enum:\n",
    "            print(\"Batch Normalization: \", batch)\n",
    "            print(\"Technique \", technique_dict[tech])\n",
    "            print(\"drop_out \", drop_out)\n",
    "            if tech == 0:\n",
    "                model,history = CNN_model_1(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out,Batch_Norm=batch)\n",
    "            elif tech == 1:\n",
    "                model,history  = CNN_model_2(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out,Batch_Norm=batch)\n",
    "            elif tech == 2:\n",
    "                model,history  = CNN_model_3(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out,Batch_Norm=batch)\n",
    "            models.append(model)\n",
    "            histories.append(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dropout 0.35 ,technique 0 ,batch True\n",
      "validation acc 0.9948\n",
      "training acc 0.9954\n",
      "dropout 0.35 ,technique 0 ,batch False\n",
      "validation acc 0.9931\n",
      "training acc 0.9893\n",
      "dropout 0.35 ,technique 1 ,batch True\n",
      "validation acc 0.9957\n",
      "training acc 0.9983\n",
      "dropout 0.35 ,technique 1 ,batch False\n",
      "validation acc 0.9936\n",
      "training acc 0.9947\n",
      "dropout 0.35 ,technique 2 ,batch True\n",
      "validation acc 0.9958\n",
      "training acc 0.9981\n",
      "dropout 0.35 ,technique 2 ,batch False\n",
      "validation acc 0.9943\n",
      "training acc 0.9953\n",
      "dropout 0.4 ,technique 0 ,batch True\n",
      "validation acc 0.9927\n",
      "training acc 0.9937\n",
      "dropout 0.4 ,technique 0 ,batch False\n",
      "validation acc 0.9939\n",
      "training acc 0.985\n",
      "dropout 0.4 ,technique 1 ,batch True\n",
      "validation acc 0.9956\n",
      "training acc 0.9983\n",
      "dropout 0.4 ,technique 1 ,batch False\n",
      "validation acc 0.9946\n",
      "training acc 0.9881\n",
      "dropout 0.4 ,technique 2 ,batch True\n",
      "validation acc 0.9955\n",
      "training acc 0.9979\n",
      "dropout 0.4 ,technique 2 ,batch False\n",
      "validation acc 0.9942\n",
      "training acc 0.9908\n",
      "dropout 0.45 ,technique 0 ,batch True\n",
      "validation acc 0.994\n",
      "training acc 0.9931\n",
      "dropout 0.45 ,technique 0 ,batch False\n",
      "validation acc 0.9943\n",
      "training acc 0.9821\n",
      "dropout 0.45 ,technique 1 ,batch True\n",
      "validation acc 0.9956\n",
      "training acc 0.9969\n",
      "dropout 0.45 ,technique 1 ,batch False\n",
      "validation acc 0.9942\n",
      "training acc 0.966\n",
      "dropout 0.45 ,technique 2 ,batch True\n",
      "validation acc 0.9962\n",
      "training acc 0.9968\n",
      "dropout 0.45 ,technique 2 ,batch False\n",
      "validation acc 0.9946\n",
      "training acc 0.9801\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "for drop_out in drop_outs:\n",
    "    for tech in techniques:\n",
    "        for batch in batch_enum:\n",
    "            print(i)\n",
    "            print(\"dropout\",drop_out, \",technique\",tech,\",batch\",batch)\n",
    "            val_acc = histories[i].history['val_acc'][-1]\n",
    "            trn_acc = histories[i].history['acc'][-1]\n",
    "            print(\"validation acc\", np.round(val_acc,4))\n",
    "            print(\"training acc\", np.round(trn_acc,4))\n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 0.99625\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_46 (Dense)             (None, 256)               200960    \n",
      "_________________________________________________________________\n",
      "activation_46 (Activation)   (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_47 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "activation_47 (Activation)   (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_48 (Dense)             (None, 10)                2570      \n",
      "_________________________________________________________________\n",
      "activation_48 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 269,322\n",
      "Trainable params: 269,322\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "val_acc_list = []\n",
    "for history in histories:\n",
    "    val_acc_list.append(history.history['acc'])\n",
    "val_acc_np = np.array(val_acc_list)\n",
    "arg = np.argmax(val_acc_np,axis = 0)[-1]\n",
    "print (arg,val_acc_np[3,-1])\n",
    "models[3].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_submission(models[-2],test_x,'sub_adam_cnn_2c_1p_2c_1p_train9965_valid9962_batch_lrannealing.csv',['ImageId','Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xl81NW5+PHPkz2BJGRjS9gFWQRR\nU5e6gFYrel266AWtu/fS1uL1tvW22lpFq1V7W9v6q0vVuta6XysqVnGh1QpCEAQEgbAIYQ3ZyT4z\nz++P800YwmRmSDIkgef9es0r893OnDPi95lzzvecI6qKMcYY01Fx3Z0BY4wxvZsFEmOMMZ1igcQY\nY0ynWCAxxhjTKRZIjDHGdIoFEmOMMZ1igcQcNkRkk4ic6b3/mYg81t15ijURmS0if4ny3Pki8h+x\nzpM59CR0dwaM6Q6q+qvuzoMxhwqrkRhzEIlIfHfnwZiuZoHEHJaCm3xEZLiIqIhcKSKbRWS3iPw8\n6Nw4EblJRNaLSJmIvCgi2UHHXxKRHSJSJSL/FJEJQceeFJGHRGSuiNQCp4fIy3wRuVNEPhaRPSLy\nuojkiMizIlItIotFZHjQ+V/19lV5f78adGyEiPxDRGpEZB6Q2+azTvQ+p1JEPhORqV3yhZrDmgUS\nY/Y6BTgS+Bpwq4iM8/b/F/ANYAowGKgAHgi67i1gNNAf+BR4tk26lwJ3AenAR+189gzgciAfGAUs\nAJ4AsoHVwG0AXgB7E7gfyAHuA94UkRwvnb8CS3AB5JfAlS0fICL53rV3euneCLwiInmRvhhjwrFA\nYsxet6tqvap+BnwGHO3t/y7wc1UtUdVGYDZwkYgkAKjq46paE3TsaBHJDEr3NVX9l6oGVLWhnc9+\nQlXXq2oVLjCtV9V3VdUHvAQc4533b8A6VX1GVX2q+hzwBXC+iAwFvgL8QlUbVfWfwOtBn3EZMFdV\n53p5mQcUAed2+BszBgskxgTbEfS+DujrvR8GvOo1B1Xiagh+YICIxIvIPV6zVzWwybsmuElpSxSf\nvTPofX2I7Za8DAa+bHPtl7iazGCgQlVr2xxrMQy4uKUcXllOAQZFkT9j2mVPbRkT2RbgGlX9V9sD\nInI5cCFwJi6IZOKaviTotK6cYnsbLiAEGwr8HdgOZIlIn6BgMjTo87cAz6jqf3ZhfoyxGokxUXgY\nuEtEhgGISJ6IXOgdSwcagTIgDYj1Y8VzgTEicqmIJIjIdGA88IaqfolrqrpdRJJE5BTg/KBr/4Jr\nAjvbq0mliMhUESmIcZ7NIc4CiTGR/QGYA7wjIjXAQuAE79jTuOajrcAq71jMqGoZcB7wY1zw+glw\nnqru9k651MtbOa6D/umga7fgak8/A0pxNZT/we4DppPEFrYyxhjTGfZLxBhjTKdYIDHGGNMpFkiM\nMcZ0igUSY4wxnXJYjCPJzc3V4cOHd3c2jDGmV1myZMluVY04hc5hEUiGDx9OUVFRd2fDGGN6FRFp\nO4tCSNa0ZYwxplMskBhjjOkUCyTGGGM6xQKJMcaYTolpIBGRaSKyRkSKReSmEMeHich7IrLcWyWu\nIOiYX0SWea85QftHiMgnIrJORF4QkaRYlsEYY0x4MQsk3trUDwDn4GYnvURExrc57TfA06o6CbgD\nuDvoWL2qTvZeFwTtvxf4naqOxk3XfW2symCMMSayWNZIjgeKVXWDqjYBz+NmHg02HnjPe/9BiOP7\nEBEBzgBe9nY9hVsC1RhjTDeJZSDJZ9+V4Uq8fcE+A77tvf8mkB609nSKiBSJyEIRaQkWOUClt/xo\ne2kCICIzveuLSktLO1sWY7rMgm0LWLl7ZXdnw5guE8sBiRJiX9s5628E/igiVwH/xK3p0BIkhqrq\nNhEZCbwvIiuA6ijSdDtVHwEeASgsLLS58g9Tu+t30zexLykJKagqrlLr1DTVkJ6UDoAv4CMhzv3v\nUNtci6qSlpjGvC/ncWz/Y/lkxydsrdnKWcPPYtH2RUzuP5mx2WNp8DUw78t5ZCRlsLJsJf6An+sm\nX8eqslV8sOUDVJVpI6ZRVl/G5prN9Ensw88/+jkA5488nx8X/pic1Jz9M25MLxLLQFICDAnaLsAt\nE9pKVbcB3wIQkb7At1W1KugYqrpBROYDxwCvAP1EJMGrleyXpuk9gm/s1U3VNPmbyE11S53X++rx\nB/z0TXJLlRdXFJOdms3m6s1kpWQRL/GU1peycNtCnvviOX5x0i/IS81j255tvLDmBY7OO5qC9AJ+\nufCXjM8Zz0VjLuLZVc/S4G+gqrGKJn8TTYEmJuZOpKKhgh21O8hOzSagAaoaq0iOT2ZP85798vzH\nZX9sfZ+akEq9r36/cx5d8eg+239e+eeQ5X99w+tUNFbw4Nce3CfAGdPbxGxhKxFJANYCX8PVNBYD\nl6rq50Hn5ALlqhoQkbsAv6reKiJZQJ2qNnrnLAAuVNVVIvIS8IqqPi8iDwPLVfXBcHkpLCxUmyKl\n4zZWbaSysZJj+h+zz35VxRfwkRifiKpStLOI4spiCvoWEB8Xz7qKdQzuO5gpBVMQEe4ruo91FesY\nkz2Goh1FlOwp4eoJV/PJ9k/4ZMcnAMycNJOJuRO57ePbKG8oZ1CfQWyv3R7T8h3b/1iWly6nb1Jf\nBqQNYHTWaNZWrGVtxVpGZY4iPz2frTVbuemEm1i0fRH5ffOpba7lvc3v8emuTxmTNYZRmaM4f9T5\nbKjawMtrX2Z4xnD++7j/5pHlj7CqbBXTj5xOXloef139V3516q/ITMrk3c3vkp2SzWkFp8W0fMZ0\nlIgsUdXCiOfFcoVEETkX+D0QDzyuqneJyB1AkarOEZGLcE9qKa5p6wde8Pgq8CcggOvH+b2q/tlL\ncySu4z4bWApcpqqN4fJhgSS8gAZo8DUgIjT6Gvmy5ksm5k5kQ+UGdtXv4rvzvgvAkPQhfHv0t/mi\n/AsqGivYXbeb9VXrGZo+FF/Ax7barq8cjsocBcD22u3U+eoYlTmKsTlj6Z/Wn01Vm0iOT+bL6i85\nfejpjMgcQYOvgRMGnsBr619jQNoAhmUMo39afwDy0vJo9Dcyb9M8Tis4jbTENPzqJz0xHRHh7rmr\nqW/2c/sFE1CFBp+PmsZ6BqRnAC5wLthQxrFDs0hJjKfZH6C20UdaUgIfrNlFQpwwblAGX5bVcVR+\nBqmJ8dQ0+PCr8uby7fgCyuQhmWyvasDnV+LjhGE5aUwq6Nfl35sxXaFHBJKe4nAIJP6AnziJo95X\nT1VjFSLCwD4DAdhRu4P7P72fT3d9Skp8Cv828t94f/P7JMUnMbDPQOZunLtfetkp2ZQ3lLf7eX0T\n++7X9HP6kNP5YMsHjMgcwfQjpzMmawyrylbx6rpXGZoxlLOGncVrxa8xqO8gzh95Pkf2m8j7JX9n\n0Y5FvLHhDX475bfsrPKzu3kNl4ybQWpcP15btoMxA9KJE6GmqYrTjhhGSUU96SkJ7KhqoMkfoLKu\niQmDMympqGf3nkb+8O46AK46eTgJca7JaHtVA7l9k1hfWsvSzRX0TU5g0cZyapv8AMTHCf6A+3/h\njLH9+WjdbrL6JLJ7TxP905MZmdeH8tpmVm933XT5/VLZ0+ijqr653e9IBCL973XCiGxe+O5J4U8y\npptYIAlyqAWSbXu2saFqA4lxieT3zefBZQ/y+obXSYxLpDmw98Y2sM9A8lLzWF2+Gl/AFybFvSbl\nTeKkQSdRXFnM4L6DyUzKJCEugSn5p5OZks6P5v+I/5g4k3E5R7K5ejPjsiYTF+ejvtnHnnohLa2e\nfsn9iCOexZvK+ee6Uuau2MHJR+Tg8ytrd9bwb5MGs6W8jic/3sQpR+SSkhjHJ1uX0ViXT5PP/Xuc\nVJBJeW0TJRX790F0hT5J8Zw0Kod3V+8Ke15CnDC4XyoJ8cKW8jqa/Upu32ROHJlNWlI8WWlJ1Df7\n2bi7lnOOGsT7X+zk3dW7yO+XyrSjBlJa08iijeXc9c2jWLC+jPLaJmqbfEwq6McR/fsyflAGQ7LT\nYlJGYzrLAkmQ3hhIdtTu4Ocf/Zz0pHTOH3k+R2QdwctrX+bJz59s95opBVPwq5+81Dw2Vm1kfeV6\n8tPzGZs9liP6HcGa8jVMyJ1AdnI2Jw2awvrq1fRN7MvgPvnsrFJU4VdzVzOpoB+VdU3UNfn5x9pS\nkhPj2FndSJMvAEBqYjyFw7P49MsK6pr95PdLpa7JT3ltEwApiXE0NAeiKmfLuVOPzKOirpmVW6ta\nawYjc/uQl55MSmI8V5w0jDvfXM32qnq+Mjybz7dVU17bRHafJG47fzxPfbyJrLQkzp04iAafn5NH\n5bKzuoHy2iay+iTR6Auwo6qeJl+Ak4/IZUh2GonxcVTVN/PYhxv47pRRJMQJVfXN7KhqICM1kcH9\nUkhOiG/NayCgxMXJfk9/tVW2p5HM1EQS4t3T9ZHON6anskASpKcGks3Vm7l70d0kxSUxqO8grpt8\nHbVNtby49kUeW/FYu9f1S+7HnSffyZsb3qS8oYLJGd/g+PyJDO/Xn7z0ZESEN5dv55VPt3B0QRZ1\nTT62VTXwxfZqdtU0Ut/kp8kfIC89GVXYvaf9LqY4gczURIbm9OGzLZURy3TSyByOHJjOB2t2ccKI\nbH4ybSzxIlzy6EKmjMnj2lNHsK2ygbSkeEbl9SWgypodNUwYnNF6s91Z3cCAjJT90vYHlGZ/gJTE\n/W/uxpiuZ4EkSE8KJO9tfo/5W+ZT3lDOmvI17Kzb2XosKc5NG9YUaGJKwRTOHnYei1flUpX0PuvK\ntlBalkMeUzlueBprtvvJ7ZvEZ1uq2Fq5b/PPiNw+bNxdu8++5IQ4Gr0axRlj+xMnUFbbxGdbKgko\nnDluAGMHpjOxIBOfX6msb+L8oweTkZLYmkZVXTMbdu/BH1BWb69m6pH9ye2bTJM/QH2Tn4GZ+9/8\nWwQCigj2y9yYXsQCSZDuDCQ1TTVs3bOV19e/TnlDOW9seAOAPol9qPfVE9AAeYEzSI3PxBe3m+0V\nAWp2fRVtziE1MZ76Zn/IdPPSkynb08jEgn6MG5jumnHqmhmYkUyjL8DgfqmMH5TBLX9bycOXHcu0\nowaFvJmX1jSSkhhHelDAMMYYiD6QHBZL7R5sFQ0VvLPpHVaWreRvxX9r3S8I1BQy2HcZEwbl8H8r\ni0ETqNH9b+I5fZIY3C+V1MR4Zhw/hIyURKYcmcfuPY34/MqQ7DQafX6S4uPa/ZWvqkz/yhASvbb6\nUE1AeenJXVRqY8zhygJJF9pUtYl3N7/LS2teah1T0Scuj+raZDKrr2V7pUAghTU0sGbHVkbm5nJx\n4RBOGpXDox9u4LbzxpOSFI8AAXV9E20NykxtfR/cERyKiJAYb01JxpjYskDSBQIa4KHPHuLhzx72\ndiTj33M0DbtPp6bRjeVI7ZNETio8dc3xjB7Qly+21zB+cEZrbeGBS4/truwb0zs1VENyuhuwE44q\n+BogMRX27IKEZGisgYx8qCpxx/oOgKZaSB8IZeshJROqS6D/BPA3Qfl6dzw5Azb+E4aeCKtfh8Zq\niEt0aWcNd9v+Jne9rwlKv4C4BBj2VUjLho0fQnJf2L4cMgZDXDxs/gTiEyEhBfrkQc12GH4KNNdD\n7S6o3gbZo6B8g8vngAlu37alsHstNFTC8FNdOZrrXB4TUqBuN6Rmwbm/gZxRMf1PYX0knbS6bDX/\n/sa/t27Xl1yGb89oTj0in8Jh2XxlRBbHDMkiJdEFDOtsNj2Sv9nd8Fr+fTbVQlIf9756O5Stg8HH\nAgqVW9yNctcqqNriblqDjoaVr0DuGHczqypxN7SAD3JHuzQqv3Q33Jod7vqcUVC+EXZ+DlnDwO9z\n52QWuLxsWQS+esg9EvoNhfpyqPjSHava4m64Kf0g4HfXVG9z6SamgAZc0PA3uUDiDzv5xV7xSe6a\nA5GQ4m7i7aaZHPnzE9Pc9waufKHGfWUOcd9doBkQGDzZlX/LImiudWUfMNHlv7neBa76Crj8VeiT\ne2Bl8lgfSYyVN5Rz9d+vZkPVBgD8jf1p2DqD/zp1CteeMsI6r03HqEJd2b7/4zfVQuMe90u2ucH9\nUpY4d+PMLIBdq2HPTkjNhtpSaKhyN/L4RHeTTR8ApWugvtL9As4e6c6p2AhJfaFys7sx98lzN73m\nOnfTTs121zdWxa68Eg+Z+bBhPqCQPtj90m6qdUFn+Kkub1uL3E1e1f3yH3ayK2NcAqT2cwGw/1iX\npr/Z1Rr6DYVBk106vnrwNbp9jTXuOy5b737dZ+S7m3NSH3csa4T7Hqu3uu85s8DtS0xzv/5Ts2HX\n5zBwkkt76Emw5ZO9eYtLcMczh7iXv8kFy8YayB7hakRVJe5z/U3ue0/JcDf/7Z9BfiFUbXZlSc12\n6TXVuvOaalyQTM6EOG8VkPoKSEqH+O67nVuNpANUlevevpmPdr6J+lMZxX9y97TpjBnQt3UQmjkM\nBALuBlWzw/26rtzibnp15e7XYs1O90u1rBg2L3Q3ocHHuJvLR79zv+J3LHc3n+R0d3Pf+qlrkug7\n0N0ga3ZA0x7vF6rQzqoJkSVn7m0Gqt7mboDZI91NKCEFCgrdDUrV/e03BHasgPRB7lVX5moA+ce6\nG6qvwd30s0fA5gV7m4qSM1x5qrZAzhEuIG5Z5NIcMcV9bkKyu9m3BKvEFPdd1le4X9Hgzo8L3wcY\nlt/nrrcWgE6xx3+DdGUgqWv0cd5ff0xp3Ps07p7CCf0u59ErCvcZJGd6oMot7hdn+QZ3A68rcy9/\ns/u7Z6d7n9oPSte6e/bnr7lrAz4oXe0CQEKKuzm11AxUAXW/rDX0o9oRZQ5xN16AcefDts/cL9Kc\n0TD0BC9gNbgmIl+ju0EnprpX9TZX0xh3vvv8unIYNMmVZfNCd25yX/erukXA735p203WRGBNWzGg\nqtw8/38pjXufwZzNrDN/yBljB1gQiSVVV/1PSHY30YZqd1MHd9NsrIba3a49vE+O68xs2uM6MkvX\nQlqOazZY+/e9abbXBt2eQZPdjbeuHEad7q4HmHCh+9yKL93Nu09/yBjkbu7J6e4XemKqu4mnZrsg\nVVUCO1fCiNNcepkFrmyBgKuxpHnNSaqdb6oYd17o/Z35pW9MCBZIDsBjRW/z/o6/Io1Def2ae0hK\nsK+vw6pKXDv06jdcJ+Hav3vNJYNc7aBsvatB+JvcdlqOu/FG07ST2Mc1EQ04yv3qrgxa8fmoi9yT\nOf2GuuAQ8Lm8DD/FPRHTUOnavGt2uM8K+PfWBBK6YMxNzqi9T9CkZu3dHxe3t1nHbvSml7E74QH4\n3SdPE5eaxoX9f2lBpIWq6wPokwdr5rraQVqO6zBtrnNNSf2Gwrp5ruklOcN1mlZsdO3gLfpPcG3w\n2z9zzUcZ+a4JSgPuxl+91d14MwtcrWTI8e7G72uEAePdDR9xv+JTMl3TTnybBx78vuh/5WeP2He7\nK4KIMYeomN4NRWQa8AfcwlaPqeo9bY4PAx4H8oBy3CJVJSIyGXgIyAD8wF2q+oJ3zZPAFKDlUZKr\nVHVZLMsBsHTrl8T1XUXfxin8ZNrEWH9cz+FrcjWFHStcgNizE1a9Bps+cjWG+goXFEJJSHXNQDtX\nwYhT9z5a2VjtmnZEYNJ0l86Ao1zgUXVt+p3VNohAtz7VYsyhLGb/Z4lIPPAAcBZu/fbFIjJHVVcF\nnfYb4GlVfUpEzsCtlng5UAdcoarrRGQwsERE3lbVluln/0dVX45V3kO5b9FjgPKHc6/bZyLDQ0JD\nFewudk/f7NnhmoYqNkLFJvfETXvNSQMnuad7TrzOPYHTd4BrttmxAo48F/oN2/uIYjRaxi0YY3qV\nWP5EOx4oVtUNACLyPHAhEBxIxgM/9N5/APwNQFXXtpygqttEZBeu1hJ5HvMY8Pn9LK+YTx+dyAlD\njuyOLBw4VRcIElLcOIGSxbDtU/doZkMV7PrC1S7ADTYLJW8cFF7jmq0GH+OCTHI69B/vnihq7xf+\nCFuD3JjDSSwDST4Q1MtJCXBCm3M+A76Na/76JpAuIjmqWtZygogcDyQB64Ouu0tEbgXeA24KtWa7\niMwEZgIMHTq0UwX5y9KPCMRXcs6Q/+xUOjG1pxRWvORu9uCanrYuCX1uYpprpho40QWZk//bPXU0\n/FQ3QM3f5M5JSDp4+TfG9FqxDCShHlJv20ZyI/BHEbkK+CewFWh9LlNEBgHPAFeqtvbM3gzswAWX\nR4CfAnfs90Gqj3jHKSws7NRgmRdWzwWN4/oTL+hMMl2jcY9rOtq1ygWKlhG6ZcXs8/VmFMCUn7oO\n6sQ01wSVf5wb/5CcDknhlne1pV+NMdGLZSApAYYEbRcA24JPUNVtwLcARKQv8G1VrfK2M4A3gVtU\ndWHQNdu9t40i8gQuGMXUzoZi+iYNJyetX6w/al/1FW5qhR0r3cjhFS+6SdpaxCe7PonEFJh4sRvD\nMOGb7hHaPv1D90+kZB68/BtjDguxDCSLgdEiMgJX05gBXBp8gojkAuVebeNm3BNciEgS8CquI/6l\nNtcMUtXt4mY//AawMoZloNHnp5Ey8lPGxvJj9tqyGL78yD0uu3nBvo/IguurmHKTCyyjz9x3LIIx\nxnSDmAUSVfWJyCzgbdzjv4+r6ucicgdQpKpzgKnA3SKiuKatH3iX/ztwGpDjNXvB3sd8nxWRPFzT\n2TLge7EqA8CG0j1IYiVD0vNj8wG+JjcYb+M/Yf17btwFuA7tU2904yZyjoD+41zQsGktjDE9TEwf\nrFfVucDcNvtuDXr/MrDfY7yq+hfgL+2keUYXZzOsJSVfIuLnyJzOddjvp2w9LH4MFj64d1/6IDjn\n127epIzBXft5xhgTIzZCK4IVOzcBcNTA4Z1PTNVNl73gASie5xbEyS9003Mce4WrcbRMk2GMMb2E\nBZIINlS4J5hHZA6JcGYEdeXwj3vhk4fdwL2pN8NxV7u1IowxphezQBJBWeMOSISBfQZ2PJGSInhu\nhpuk8Ngr4dz/tbmbjDGHDAskEdQGSkkgnbTEDoytqNgEr82CTR+6RYD+4z23gJAxxhxCLJBE0KgV\npEkH+i327IKXr3FrYpz+czjhe25dDGOMOcRYIInAp3WkxB/gbLSrXoPXb4CmOvjWn9wgQWOMOURZ\nIAmj2R8gIPWkJfSP/qL598D8u936Ghc/CXljYpY/Y4zpCSyQhFFd34zEN9A3MYoaib8Z3rsdPv5/\nMPSrcOnzNh2JMeawcACLRRx+quqbkbgG0pPSI5/8/p0uiAycCNOfsSBijDlsWI0kjJqGZohrIj05\nQiDZtdqNUD/6UvjmQwcnc8YY00NYjSSMPc17EFH6JIRZuW/PLnj8bLek7Bm3HLzMGWNMD2E1kjBq\nm2sBSEsI00fy0e+hoRquehMyYzSxozHG9GBWIwmjtrkGgLT2aiS7VrspT469AoaffBBzZowxPYcF\nkjAa/U0AJMWnhD5h3q1uadqv3XYQc2WMMT2LBZIw/N6iUvESv//BT5+Gde/AaTdCn5yDnDNjjOk5\nLJCEEQh4gSSuzWJSAT/88zdQcDyc9IMQVxpjzOEjpoFERKaJyBoRKRaRm0IcHyYi74nIchGZLyIF\nQceuFJF13uvKoP3HicgKL837vSV3Y8KnfgASpM3XVPQ4VH4JX70e4kLUVowx5jASs0AiIvHAA8A5\nwHjgEhEZ3+a03+DWZZ8E3AHc7V2bDdwGnAAcD9wmIi2Lkz8EzARGe69psSqD36uRxLUNFp8+DfnH\nuZUMjTHmMBfLGsnxQLGqblDVJuB54MI254wH3vPefxB0/GxgnqqWq2oFMA+YJiKDgAxVXaCqCjwN\nfCNWBQh4fSQJwYFk9zrYsRzGX2jrpxtjDLENJPnAlqDtEm9fsM+Ab3vvvwmki0hOmGvzvffh0gRA\nRGaKSJGIFJWWlnaoAK01kuCA8cnDkJACR1/SoTSNMeZQE8tAEurnurbZvhGYIiJLgSnAVsAX5tpo\n0nQ7VR9R1UJVLczLy4s+10F8AddHEh8X9DUVvwejvgZ9D2BGYGOMOYTFMpCUAMELnRcA24JPUNVt\nqvotVT0G+Lm3ryrMtSXe+3bT7EqtTVviTQCwuxgqNsKIU2P1kcYY0+vEMpAsBkaLyAgRSQJmAHOC\nTxCRXJHWR6JuBh733r8NfF1EsrxO9q8Db6vqdqBGRE70nta6AngtVgVoHUfS8vjvp0+6ObXGx6xb\nxhhjep2YBRJV9QGzcEFhNfCiqn4uIneIyAXeaVOBNSKyFhgA3OVdWw78EheMFgN3ePsAvg88BhQD\n64G3YlUGf+s4Eu9rWj8fhp0MGYNi9ZHGGNPrxHTSRlWdC8xts+/WoPcvAy+3c+3j7K2hBO8vAo7q\n2pyG5vfGkcRLPDRUwc6VMHW/4TDGGHNYs5HtYQTU9eMnxMXDjpWAuvEjxhhjWlkgCcMfPEVK6Wq3\ns/+4bsyRMcb0PBZIwggET9q46wtIzoAMW3PEGGOCWSAJo7WPJC4OytZB7mgbzW6MMW1EDCTe6PAf\nBM11ddhoqZEkxsVD+QbIHtXNOTLGmJ4nmhrJDGAwsFhEnheRs2M5425P0jqOJOCDqhLIHtnNOTLG\nmJ4nYiBR1WJV/TkwBvgr7pHczSJyuzdL7yFLvUCSVLcTNGCBxBhjQoiqj0REJgG/Bf4XeAW4CKgG\n3o9d1rqfz3tqK7Fmq9uRY01bxhjTVsQBiSKyBKgE/gzcpKqN3qFPROTkWGauu7X0kSRVexMOW43E\nGGP2E83I9otVdUOoA6r6rS7OT4/SOmljzRZIyYTUw+55A2OMiSiapq3/EJF+LRveRIp3xjBPPUbr\nOJKaHZA51B79NcaYEKIJJOeoamXLhrdi4bmxy1LP0RpIGioh7ZB+rsAYYzosmkASLyLJLRsikgok\nhzn/kNH6+G9DlQUSY4xpRzR9JH8B3hORJ3CrEV4DPBXTXPUQGlwjSbVAYowxoUQMJKr6axFZAXwN\nt9TtL1X17ZjnrAdoqZHENVRaR7sxxrQjqvVIVPUtYriAVE/V0kcSFwhY05YxxrQjmrm2ThSRxSKy\nR0SaRMQvItXRJC4i00RkjYgUi8h+K0KJyFAR+UBElorIchE519v/HRFZFvQKiMhk79h8L82WY/0P\ntNDRau1sB6uRGGNMO6KpkfxQ2fGdAAAgAElEQVQRN9/WS0Ahbp30IyJdJCLxwAPAWUAJbq6uOaq6\nKui0W3BL8D4kIuNxqykOV9VngWe9dCYCr6nqsqDrvuOtlBhTLQtbCVgfiTHGtCOqKVJUtRiIV1W/\nqj4BnB7FZccDxaq6QVWbgOeBC9smDWR47zOBbSHSuQR4Lpp8drXWPhLUaiTGGNOOaGokdSKSBCwT\nkV8D24E+UVyXD2wJ2i4BTmhzzmzgHRG53kvzzBDpTGf/APSEiPhx837dqepVHYKIyExgJsDQoUOj\nyO7+Wp7ailOsj8QYY9oRTY3kcu+8WUAtMAT4dhTXhRoG3vaGfwnwpKoW4AY5PiMirXkSkROAOlVd\nGXTNd1R1InCq97o81Ier6iOqWqiqhXl5eVFkd38BWmokWI3EGGPaETaQeP0cd6lqg6pWq+rtqvoj\nr6krkhJc0GlRwP5NV9cCLwKo6gIgBcgNOj6DNs1aqrrV+1uDm9b++Cjy0iGtT20BpPQLe64xxhyu\nwgYSVfUDeV7T1oFaDIwWkRHe9TOAOW3O2Ywbn4KIjMMFklJvOw64GNe3grcvQURyvfeJwHnASmKk\nJZBIcibER/WktDHGHHaiuTtuAv4lInNwTVsAqOp94S5SVZ+IzALexj1B+7iqfi4idwBFqjoH+DHw\nqIj8ENfsdVVQf8dpQEmbmYeTgbe9IBIPvAs8GkUZOkQ14PWPWLOWMca0J5pAss17xQHpB5K4qs7F\nPdIbvO/WoPergJBrmqjqfODENvtqgeMOJA+dESDgOnpSMg/WRxpjTK8TzRQptx+MjPREAVUXSBJS\nuzsrxhjTY0WzQuIH7P+0Fap6Rkxy1IO0Nm0lHBaTHRtjTIdE07R1Y9D7FNyjv77YZKdnCWjAPY2Q\naDUSY4xpTzRNW0va7PqXiPwjRvnpUQKqblR7Qkp3Z8UYY3qsaJq2god0x+E6uwfGLEc9iNLStGWB\nxBhj2hNN09YSXB+J4Jq0NuIGEh7yXNOWQqIFEmOMaU80TVsjDkZGeiIl4KaQtxqJMca0K5r1SH4g\nIv2CtrNE5LrYZqtnUFXi1PpIjDEmnGgmbfxPVa1s2VDVCuA/Y5elnkPVb53txhgTQTSBJE5EWmfy\n9SZy7MjcW72P+tyAROsjMcaYdkXT2f428KKIPIzrdP8e8PeY5qqnUJ/XR2LjSIwxpj3RBJKf4haI\n+j7uya13gMdimameQvDbyHZjjIkgmkCSCjyqqg9Da9NWMlAXy4z1BKLNCGoj240xJoxo+kjewwWT\nFqm46dsPeUcOSHNfkNVIjDGmXdEEkhRV3dOy4b1Pi12Weo7keHVNW/EWSIwxpj3RBJJaETm2ZUNE\njgPqo0lcRKaJyBoRKRaRm0IcHyoiH4jIUhFZLiLnevuHi0i9iCzzXg8Hf76IrPDSvD/4ibKu1jpp\no0TzNRljzOEpmj6S/wZeEpGW9dYHAdMjXeT1pTwAnIVbv32xiMzxFrNqcQvwoqo+JCLjcYtgDfeO\nrVfVySGSfgjX+b/QO38a8FYU5ThgrVOkWCAxxph2RTNFymIRGQsciXtq6wtVbY4i7eOB4palckXk\neeBCIDiQKJDhvc/ErcTYLhEZBGSo6gJv+2ngG8QykCgQu0qPMcb0etHUSMAFkfG49UiOERFU9ekI\n1+QDW4K2S4AT2pwzG3hHRK4H+gBnBh0bISJLgWrgFlX90EuzpE2a+aE+XERm4mouDB06NEJWQwug\nXtufBRJjjGlPNHNt3Qb8P+91OvBr4IIo0g5192270uIlwJOqWgCcCzwjInHAdmCoqh4D/Aj4q4hk\nRJmm26n6iKoWqmphXl5eFNkNlYa3ZrvFEWOMaVc0jf8XAV8Ddqjq1cDRuHEkkZQAQ4K2C9i/6epa\n4EUAr7kqBchV1UZVLfP2LwHWA2O8NAsipNll/NZHYowxEUVzh6xX1QDg82oFu4CRUVy3GBgtIiNE\nJAmYAcxpc85mXJBCRMbhAkmpiOR5nfWIyEhgNLBBVbcDNSJyove01hXAa1HkpUMuyZ/KtZXVWJXE\nGGPaF00fSZE3jfyjuEWu9gCLIl2kqj4RmYWbqyseeFxVPxeRO4AiVZ0D/Bh4VER+iGuiukpVVURO\nA+4QER/gB76nquVe0t8HnsQNjHyLGHW0A5yWfRTU1VtnuzHGhBHNU1sta488LCJ/xz01tTyaxFV1\nLu4R3eB9twa9XwWcHOK6V4BX2kmzCDgqms/vvJbuFwskxhjTnmif2gJAVTfFKB89k3qBxPpIjDGm\nXXaHDEcD7q81bRljTLsskIRlTVvGGBNJxKYtEckOsbsmytHtvVtr05YFEmOMaU80NZJPgVJgLbDO\ne79RRD71JnA8hFkfiTHGRBLNHfLvwLmqmquqOcA5uEGE1wEPxjJz3a6lj8Satowxpl3RBJJCVX27\nZUNV3wFOU9WFRDfCvfdq7SKxQGKMMe2J5vHfchH5KfC8tz0dqPBGngfav+xQYJ3txhgTSTQ1kktx\nc1r9DTcdyVBvXzzw77HLWg9g40iMMSaiaEa27waub+dwcddmp4dpHUfSvdkwxpieLJrHf8cAN+JW\nLmw9X1XPiF22egpr2jLGmEii6SN5CXgYeAw3geLhw8aRGGNMRNEEEp+qPhTznPRI1kdijDGRRHOH\nfF1ErhORQSKS3fKKec56AhtHYowxEUVTI7nS+/s/QfuU6Ba36t2sacsYYyKK5qmtEQcjIz2TdbYb\nY0wk7QYSETlDVd8XkW+FOq6q/xcpcRGZBvwBN+bkMVW9p83xocBTQD/vnJtUda6InAXcAyQBTcD/\nqOr73jXzgUFAvZfM11V1V6S8dIiNIzHGmIjC1UimAO8D54c4pkDYQOKNfH8AOAsoARaLyBxvVcQW\ntwAvqupDIjIet5ricGA3cL6qbhORo3DL9eYHXfcdb6XE2LL1SIwxJqJ2A4mq3ub9vbqDaR8PFKvq\nBgAReR64EAgOJApkeO8zgW3eZy4NOudzIEVEklW1sYN56SBr2jLGmEiiGZCYDHyb/Qck3hHh0nxg\nS9B2CXBCm3NmA++IyPVAH+DMEOl8G1jaJog8ISJ+3Lrud6q2tEHtk++ZwEyAoUOHRshqO6yz3Rhj\nIoqm8f81XE3CB9QGvSIJdfdte8O/BHhSVQuAc4FnRPZ2SIjIBOBe4LtB13xHVScCp3qvy0N9uKo+\noqqFqlqYl5cXRXbDsD4SY4xpVzSP/xao6rQOpF0CDAlOB6/pKsi1wDQAVV0gIilALrBLRAqAV4Er\nVHV9ywWqutX7WyMif8U1oT3dgfxFZuNIjDEmomh+an8sIhM7kPZiYLSIjBCRJGAGMKfNOZuBrwGI\nyDggBSgVkX7Am8DNqvqvlpNFJEFEcr33icB5wMoO5C061rRljDERRVMjOQW4SkQ2Ao24n+eqqpPC\nXaSqPhGZhXviKh54XFU/F5E7gCJVnQP8GHhURH6Ia/a6SlXVu+4I4Bci8gsvya/jmtTe9oJIPPAu\n8OgBlvkA7Nf1Yowxpo1oAsk5HU1cVefiHukN3ndr0PtVwMkhrrsTuLOdZA/eOvE2jsQYYyIKNyAx\nQ1WrgZqDmJ+excaRGGNMROFqJH/F9UEswbXxBN9ND4+5tmwciTHGRBRuQOJ53t/Dd64t62w3xpiI\noukjQUSygNG4p6oAUNV/xipTPYf1kRhjTCTRjGz/D+AG3DiQZcCJwALg0F9q18aRGGNMRNH81L4B\n+ArwpaqeDhwDlMY0Vz2FNW0ZY0xE0QSSBlVtADfvlqp+ARwZ22z1FNbZbowxkUTTR1LijTT/GzBP\nRCrYf6qTQ5PVSIwxJqJoVkj8pvd2toh8gJvu/e8xzVVP0TqOxDrbjTGmPWEDiTcT73JVPQpAVf9x\nUHLV41iNxBhj2hP2p7aqBoDPvCVxDz/WtGWMMRFF00cyCPhcRBYRtA6Jql4Qs1z1GNbZbowxkUQT\nSG6PeS56KquRGGNMRNEEknNV9afBO0TkXuDQ7y+xSRuNMSaiaB5HOivEvg5PLd+7WNOWMcZEEm4a\n+e8D1wEjRWR50KF04F+hrzrEWNOWMcZEFK5G8lfgfNzyuOcHvY5T1cuiSVxEponIGhEpFpGbQhwf\nKiIfiMhSEVkuIucGHbvZu26NiJwdbZpdyyZtNMaYSMJNI18FVAGXdCRhEYkHHsA1jZUAi0Vkjrcq\nYotbgBdV9SERGY9bTXG4934GMAEYDLwrImO8ayKl2XVs0kZjjIkolj+1jweKVXWDqjYBzwMXtjlH\ngQzvfSZ7p165EHheVRtVdSNQ7KUXTZpdx5q2jDEmolgGknxgS9B2ibcv2GzgMhEpwdVGro9wbTRp\nAiAiM0WkSESKSks7OlmxdbYbY0wksQwkoe6+2mb7EuBJVS0AzgWe8aZlae/aaNJ0O1UfUdVCVS3M\ny8s7gGzvk4j7a30kxhjTrqhWSOygEmBI0HYB+88afC0wDUBVF4hICpAb4dpIaXYdG0dijDERxfKn\n9mJgtIiMEJEkXOf5nDbnbAa+BiAi43BL+ZZ6580QkWQRGYFb5ndRlGl2IWvaMsaYSGJWI1FVn4jM\nAt4G4oHHVfVzEbkDKFLVOcCPgUdF5Ie4u/ZVqqq4ub1eBFYBPuAHquoHCJVmrMqwN45YIDHGmPbE\nsmkLVZ2L60QP3ndr0PtVwMntXHsXcFc0acaO9ZEYY0wkdocMx8aRGGNMRBZIwrFxJMYYE1FMm7Z6\nPwskxvQ2zc3NlJSU0NDQ0N1Z6TVSUlIoKCggMTGxQ9dbIAlH2xu6YozpqUpKSkhPT2f48OGI/QiM\nSFUpKyujpKSEESNGdCgNa9oKRwNWGzGml2loaCAnJ8eCSJREhJycnE7V4CyQhGU1EmN6IwsiB6az\n35cFknBUrUZijDERWCAJS20MiTHGRGB3yXA0gDVtGWM6Q1UJBAKRT+wEv98f0/Qjsae2wrGmLWN6\ntdtf/5xV26q7NM3xgzO47fwJYc/ZtGkT55xzDqeffjoLFixg2bJl/OQnP+Hdd98lKyuLX/3qV/zk\nJz9h8+bN/P73v+eCCy7g888/5+qrr6apqYlAIMArr7xCYmIi06ZN44QTTmDp0qWMGTOGp59+mrS0\nNIYPH84111zDO++8w6xZsxg7dizf+973qKurY9SoUTz++ONkZWUxdepUJk+ezKJFi6iurubxxx/n\n+OOP79LvxGokYVlnuzGmY9asWcMVV1zB0qVLAZg6dSpLliwhPT2dW265hXnz5vHqq69y661u1qiH\nH36YG264gWXLllFUVERBQUFrOjNnzmT58uVkZGTw4IMPtn5GSkoKH330ETNmzOCKK67g3nvvZfny\n5UycOJHbb7+99bza2lo+/vhjHnzwQa655pouL6vVSMJR6yMxpjeLVHOIpWHDhnHiiScCkJSUxLRp\n0wCYOHEiycnJJCYmMnHiRDZt2gTASSedxF133UVJSQnf+ta3GD16NABDhgzh5JPdlISXXXYZ999/\nPzfeeCMA06dPB6CqqorKykqmTJkCwJVXXsnFF1/cmpdLLnErpp922mlUV1dTWVlJv379uqysdpcM\nx5q2jDEd1KdPn9b3iYmJrY/YxsXFkZyc3Pre5/MBcOmllzJnzhxSU1M5++yzef/994H9H80N3g7+\njHDCpdEVLJCEZU1bxpiDY8OGDYwcOZL/+q//4oILLmD58uUAbN68mQULFgDw3HPPccopp+x3bWZm\nJllZWXz44YcAPPPMM621E4AXXngBgI8++ojMzEwyMzO7NO/WtBWO1UiMMQfJCy+8wF/+8hcSExMZ\nOHAgt956K9XV1YwbN46nnnqK7373u4wePZrvf//7Ia9/6qmnWjvbR44cyRNPPNF6LCsri69+9aut\nne1dTVRDLnneNYmLTAP+gFuE6jFVvafN8d8Bp3ubaUB/Ve0nIqcDvws6dSwwQ1X/JiJPAlOAKu/Y\nVaq6LFw+CgsLtaio6MAL8NZP4bPn4KbNB36tMaZbrF69mnHjxnV3NrrEpk2bOO+881i5cmWH05g6\ndSq/+c1vKCwsDHteqO9NRJaoavgLiWGNRETigQeAs3BrsC8WkTneYlYAqOoPg86/HjjG2/8BMNnb\nnw0UA+8EJf8/qvpyrPLeysaRGGNMRLFs2joeKFbVDQAi8jxwIW753FAuAW4Lsf8i4C1VrYtJLsOx\npi1jTDcaPnx4p2ojAPPnz++azIQRy872fGBL0HaJt28/IjIMGAG8H+LwDOC5NvvuEpHlIvI7EUlu\nJ82ZIlIkIkWlpaUHnnvAOtuNMSayWAaSUHfg9jpkZgAvq+o+4/xFZBAwEXg7aPfNuD6TrwDZwE9D\nJaiqj6hqoaoW5uXlHWjeWxKxcSTGGBNBLO+SJcCQoO0CYFs754aqdQD8O/Cqqja37FDV7eo0Ak/g\nmtBiw9YjMcaYiGIZSBYDo0VkhIgk4YLFnLYniciRQBawIEQal9AmwHi1FMSNqPkG0LkGxLCsacsY\nYyKJWSBRVR8wC9cstRp4UVU/F5E7ROSCoFMvAZ7XNs8hi8hwXI3mH22SflZEVgArgFzgztiUAOts\nN8Z02F133cWECROYNGkSkydP5pNPPsHn8/Gzn/2M0aNHM3nyZCZPnsxdd93Vek18fDyTJ09mwoQJ\nHH300dx3330xnzm4K8R0QKKqzgXmttl3a5vt2e1cu4kQnfOqekbX5TAS6yMxxhy4BQsW8MYbb/Dp\np5+SnJzM7t27aWpq4pZbbmHHjh2sWLGClJQUampq+O1vf9t6XWpqKsuWuWFxu3bt4tJLL6Wqqmqf\nCRh7IhvZHo6NIzGmd3vrJtixomvTHDgRzrkn7Cnbt28nNze3dU6t3Nxc6urqePTRR9m0aRMpKSkA\npKenM3v27JBp9O/fn0ceeYSvfOUrzJ49u0cvH2w/t8NRrGnLGHPAvv71r7NlyxbGjBnDddddxz/+\n8Q+Ki4sZOnQo6enpUaczcuRIAoEAu3btimFuO89qJGFZZ7sxvVqEmkOs9O3blyVLlvDhhx/ywQcf\nMH36dH72s5/tc84TTzzBH/7wB8rKyvj4448ZMmRIyLRiOY1VV7FAEo6NIzHGdFB8fDxTp05l6tSp\nTJw4kT/96U9s3ryZmpoa0tPTufrqq7n66qs56qij2l0qd8OGDcTHx9O/f/+DnPsDY3fJcDRgFRJj\nzAFbs2YN69ata91etmwZRx55JNdeey2zZs2ioaEBcGutNzU1hUyjtLSU733ve8yaNatH94+A1Ugi\nsKYtY8yB27NnD9dffz2VlZUkJCRwxBFH8Mgjj5CZmckvfvELjjrqKNLT00lNTeXKK69k8ODBANTX\n1zN58mSam5tJSEjg8ssv50c/+lE3lyYyCyTh2DgSY0wHHHfccXz88cchj91zzz3cc0/ovpv2mrh6\nOmvaCstqJMYYE4kFknCss90YYyKyu2Q4NmmjMcZEZIEkLGvaMsaYSCyQhGOd7cYYE5EFkrCsj8QY\nYyKxu2Q4NmmjMSZGnnzySWbNmtXd2egSFkjCsaYtY4yJyAYkRmSBxJje6t5F9/JF+RddmubY7LH8\n9PifRjzvG9/4Blu2bKGhoYEbbriBmTNn8sQTT3D33XczaNAgxowZ0zrN/Ouvv86dd95JU1MTOTk5\nPPvsswwYMIDZs2ezceNGtm/fztq1a7nvvvtYuHAhb731Fvn5+bz++uskJiZ2afk6IqY1EhGZJiJr\nRKRYRG4Kcfx3IrLMe60VkcqgY/6gY3OC9o8QkU9EZJ2IvOAt4xsbNo7EGNNBjz/+OEuWLKGoqIj7\n77+frVu3ctttt/Gvf/2LefPmsWrVqtZzTznlFBYuXMjSpUuZMWMGv/71r1uPrV+/njfffJPXXnuN\nyy67jNNPP50VK1aQmprKm2++2R1F20/MaiQiEg88AJwFlACLRWSOqrZ+e6r6w6DzrweOCUqiXlUn\nh0j6XuB3qvq8iDwMXAs8FIsy2KSNxvRu0dQcYuX+++/n1VdfBWDLli0888wzTJ06lby8PACmT5/O\n2rVrASgpKWH69Ols376dpqYmRowY0ZrOOeecQ2JiIhMnTsTv9zNt2jQAJk6cyKZNmw5uodoRy5/b\nxwPFqrpBVZuA54ELw5x/CfBcuATFTYF5BvCyt+sp4BtdkNd22DgSY8yBmz9/Pu+++y4LFizgs88+\n45hjjmHs2LHtzuJ7/fXXM2vWLFasWMGf/vSn1tmBgdbmr7i4OBITE1vTiIuLw+fzxb4wUYhlIMkH\ntgRtlxBiDXYAERkGjADeD9qdIiJFIrJQRFqCRQ5Qqaot3164NGd61xeVlpZ2rATW2W6M6YCqqiqy\nsrJIS0vjiy++YOHChdTX1zN//nzKyspobm7mpZde2uf8/Hx3K3vqqae6K9sdFstAEuoO3N5SXzOA\nl1U1eOrLoapaCFwK/F5ERh1Imqr6iKoWqmphS1XywFkfiTHmwE2bNg2fz8ekSZP4xS9+wYknnsig\nQYOYPXs2J510EmeeeSbHHnts6/mzZ8/m4osv5tRTTyU3N7cbc94xEqtlHEXkJGC2qp7tbd8MoKp3\nhzh3KfADVQ0577KIPAm8AbwClAIDVdXX9jPaU1hYqEVFRQdeiA9/C401cObsA7/WGNMtVq9ezbhx\n47o7G71OqO9NRJZ4P+jDiuXP7cXAaO8pqyRcrWNO25NE5EggC1gQtC9LRJK997nAycAqdVHvA+Ai\n79QrgddiVoJTf2xBxBhjIohZIPH6MWYBbwOrgRdV9XMRuUNELgg69RLged23ajQOKBKRz3CB456g\np71+CvxIRIpxfSZ/jlUZjDHGRBbTAYmqOheY22bfrW22Z4e47mNgYjtpbsA9EWaMMSGpao9f57wn\n6WwXh/UkG2MOKSkpKZSVlXX65ni4UFXKyspISUnpcBo2RYox5pBSUFBASUkJHX7s/zCUkpJCQUFB\nh6+3QGKMOaQkJibuMzLcxJ41bRljjOkUCyTGGGM6xQKJMcaYTonZyPaeRERKgS87eHkusLsLs9Od\nrCw9z6FSDrCy9FSdKcswVY04x9RhEUg6Q0SKopkioDewsvQ8h0o5wMrSUx2MsljTljHGmE6xQGKM\nMaZTLJBE9kh3Z6ALWVl6nkOlHGBl6aliXhbrIzHGGNMpViMxxhjTKRZIjDHGdIoFkjBEZJqIrBGR\nYhG5qbvzE4mIPC4iu0RkZdC+bBGZJyLrvL9Z3n4Rkfu9si0XkWPbT/ngEpEhIvKBiKwWkc9F5AZv\nf28sS4qILBKRz7yy3O7tHyEin3hlecFb/A0RSfa2i73jw7sz/22JSLyILBWRN7zt3lqOTSKyQkSW\niUiRt6/X/fsCEJF+IvKyiHzh/T9z0sEuiwWSdohIPPAAcA4wHrhERMZ3b64iehKY1mbfTcB7qjoa\neM/bBleu0d5rJvDQQcpjNHzAj1V1HHAi8APvu++NZWkEzlDVo4HJwDQRORG4F/idV5YK4Frv/GuB\nClU9Avidd15PcgNuoboWvbUcAKer6uSgMRa98d8XwB+Av6vqWOBo3H+fg1sWVbVXiBdwEvB20PbN\nwM3dna8o8j0cWBm0vQYY5L0fBKzx3v8JuCTUeT3thVtO+azeXhYgDfgUOAE30jih7b813IqiJ3nv\nE7zzpLvz7uWnAHdTOgN4A5DeWA4vT5uA3Db7et2/LyAD2Nj2uz3YZbEaSfvygS1B2yXevt5mgKpu\nB/D+9vf294ryeU0ixwCf0EvL4jUHLQN2AfOA9UCluuWoYd/8tpbFO16FW1K6J/g98BMg4G3n0DvL\nAaDAOyKyRERmevt647+vkUAp8ITX5PiYiPThIJfFAkn7Qq3TeSg9K93jyycifYFXgP9W1epwp4bY\n12PKoqp+VZ2M+0V/PDAu1Gne3x5ZFhE5D9ilqkuCd4c4tUeXI8jJqnosrqnnByJyWphze3JZEoBj\ngYdU9Riglr3NWKHEpCwWSNpXAgwJ2i4AtnVTXjpjp4gMAvD+7vL29+jyiUgiLog8q6r/5+3ulWVp\noaqVwHxcv08/EWlZWC44v61l8Y5nAuUHN6chnQxcICKbgOdxzVu/p/eVAwBV3eb93QW8igvwvfHf\nVwlQoqqfeNsv4wLLQS2LBZL2LQZGe0+lJAEzgDndnKeOmANc6b2/Etff0LL/Cu8pjhOBqpaqcHcT\nEQH+DKxW1fuCDvXGsuSJSD/vfSpwJq4z9APgIu+0tmVpKeNFwPvqNWZ3J1W9WVULVHU47v+F91X1\nO/SycgCISB8RSW95D3wdWEkv/PelqjuALSJypLfra8AqDnZZuruzqCe/gHOBtbg27Z93d36iyO9z\nwHagGffL41pcu/R7wDrvb7Z3ruCeSlsPrAAKuzv/QeU4BVfdXg4s817n9tKyTAKWemVZCdzq7R8J\nLAKKgZeAZG9/irdd7B0f2d1lCFGmqcAbvbUcXp4/816ft/y/3Rv/fXn5mwwUef/G/gZkHeyy2BQp\nxhhjOsWatowxxnSKBRJjjDGdYoHEGGNMp1ggMcYY0ykWSIwxxnSKBRJjeiARmdoyw64xPZ0FEmOM\nMZ1igcSYThCRy7z1RpaJyJ+8CRr3iMhvReRTEXlPRPK8cyeLyEJvHYhXg9aIOEJE3hW3ZsmnIjLK\nS75v0DoTz3oj/hGRe0RklZfOb7qp6Ma0skBizP9v7+5ZowrCKI7/jwiKL0QsbCwUG5GALNj51uQL\nRDEElCDWNlrZCIr4FQQFm4gWFiI2YmOxYKWYMmWq9KKIRDA5Fs+sWKwJu3djmvOrlmHu3DvF7sNc\n2POMSdIpYJ4KAOwB68A1YD+w5AoF7AP32iXPgDu2T1P/Kh6MvwAeuXqWnKXSCaBSj29R/XBOAOck\nHQYuAdNtnYfbu8uIraWQRIxvBjgDfGox8TPUD/4G8LLNeQ6clzQFHLLdb+OLwMWW+XTU9msA22u2\nf7Q5H22v2t6gYmKOA9+ANeCppMvAYG7EjkkhiRifgEVXl72e7ZO27w+Zt1kO0bBY74Gff31epxpI\n/aKSal8Bs8C7EZ85YnuXTb4AAAC/SURBVOJSSCLG9x64IukI/On5fYz6Xg0Sca8CH2x/Bb5IutDG\nF4C+q8/KqqTZtsYeSfv+dcPWo2XK9lvqtVdvOzYWMYrdW0+JiGFsL0u6S3Xa20WlLt+kmgtNS/pM\ndQacb5dcBx63QrEC3GjjC8ATSQ/aGnOb3PYg8EbSXuo0c3vC24oYWdJ/IyZM0nfbB3b6OSL+l7za\nioiITnIiiYiITnIiiYiITlJIIiKikxSSiIjoJIUkIiI6SSGJiIhOfgP7Fdg6Ad+vlQAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x217f6510dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.94419047619 0.946976190476\n"
     ]
    }
   ],
   "source": [
    "plt.figure()\n",
    "hist1 = rmsprop_history.history['acc']\n",
    "hist2 = sgd_history.history['acc']\n",
    "hist3 = adam_history.history['acc']\n",
    "plt.title('linear model')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('training accuracy')\n",
    "plt.plot(hist1,label='rmsprop')\n",
    "plt.plot(hist2,label='SGD')\n",
    "plt.plot(hist3,label='adam')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "print(hist1[-1],hist3[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3 4]\n",
      "[[1 2]\n",
      " [2 0]\n",
      " [3 9]\n",
      " [4 7]] (28000, 2)\n",
      "   ImageId  Label\n",
      "0        1      2\n",
      "1        2      0\n",
      "2        3      9\n",
      "3        4      7\n",
      "4        5      3\n"
     ]
    }
   ],
   "source": [
    "predictions.shape\n",
    "result = predictions.argmax(axis =1 )\n",
    "result.shape\n",
    "result = result.reshape((28000,1))\n",
    "ids = (np.arange(28000)+1).reshape((28000,1))\n",
    "result = np.hstack((ids,result))\n",
    "print(result[:4,:],result.shape)\n",
    "m = to_kaggle_csv(result,['ImageId','Label'],'submission_adam_model_300ep_no_regularization.csv')\n",
    "print(m.head())\n",
    "#plt.imshow(test_x[3,:].reshape((28,28)))\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary for the linear model\n",
    "####  The output of the linear model\n",
    "the submission got the 1653th place out of 1879, this is the top 87.9% with accuracy of 91.6%, which is really bad for mnist dataset, as it's the easiest task you'll ever workon and it's far below our human level accuracy.\n",
    "Note that training accuracy is around 94.4% and the submission accuracy is around 91.6%.\n",
    "We've been training for around 270 epochs\n",
    "<img src=\"1-linearmodel.PNG\" height=\"300\" width = \"800\" >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear models with regularization\n",
    "\n",
    "now the model is overfitting --> training acc is 94.4 and submission accuracy is 91.6.\n",
    "Next step is to add regularization and reduce overfitting --> thus we need a separate validation set to monitor overfitting.\n",
    "We'll find that a linear model isn't capable of overfitting and any further regularization will only reduce the training error, as the model doesn't have the capacity to overfit\n",
    "\n",
    "That's why it's a best practice in neural networks to overfit the data first to make sure that your model is good, then add regularization to generalize better, but you shouldn't spend anytime trying to prevent overfitting using a model with high bias (simple models)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "## validation set\n",
    "y = train_data[:,0]\n",
    "X = train_data[:,1:]\n",
    "enc = preprocessing.OneHotEncoder()\n",
    "enc.fit(y.reshape((y.shape[0],1)))\n",
    "train_y = enc.transform(y.reshape((y.shape[0],1))).toarray()\n",
    "\n",
    "train_x, valid_x, train_y, valid_y = train_test_split(X, train_y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/150\n",
      "33600/33600 [==============================] - 4s 107us/step - loss: 1.5293 - acc: 0.8036 - val_loss: 1.3471 - val_acc: 0.8408\n",
      "Epoch 2/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3466 - acc: 0.8391 - val_loss: 1.3412 - val_acc: 0.8390\n",
      "Epoch 3/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3423 - acc: 0.8398 - val_loss: 1.3424 - val_acc: 0.8383\n",
      "Epoch 4/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3394 - acc: 0.8385 - val_loss: 1.3337 - val_acc: 0.8396\n",
      "Epoch 5/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3376 - acc: 0.8388 - val_loss: 1.3358 - val_acc: 0.8371\n",
      "Epoch 6/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3364 - acc: 0.8385 - val_loss: 1.3352 - val_acc: 0.8418\n",
      "Epoch 7/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3351 - acc: 0.8392 - val_loss: 1.3383 - val_acc: 0.8363\n",
      "Epoch 8/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3345 - acc: 0.8378 - val_loss: 1.3319 - val_acc: 0.8318\n",
      "Epoch 9/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3338 - acc: 0.8373 - val_loss: 1.3335 - val_acc: 0.8387\n",
      "Epoch 10/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 1.3335 - acc: 0.8389 - val_loss: 1.3334 - val_acc: 0.8426\n",
      "Epoch 11/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 1.3336 - acc: 0.8387 - val_loss: 1.3348 - val_acc: 0.8312\n",
      "Epoch 12/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3332 - acc: 0.8381 - val_loss: 1.3317 - val_acc: 0.8364\n",
      "Epoch 13/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3331 - acc: 0.8378 - val_loss: 1.3301 - val_acc: 0.8401\n",
      "Epoch 14/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3326 - acc: 0.8396 - val_loss: 1.3310 - val_acc: 0.8375\n",
      "Epoch 15/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3332 - acc: 0.8376 - val_loss: 1.3331 - val_acc: 0.8236\n",
      "Epoch 16/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3331 - acc: 0.8377 - val_loss: 1.3336 - val_acc: 0.8405\n",
      "Epoch 17/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3333 - acc: 0.8373 - val_loss: 1.3363 - val_acc: 0.8429\n",
      "Epoch 18/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3326 - acc: 0.8376 - val_loss: 1.3342 - val_acc: 0.8363\n",
      "Epoch 19/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8371 - val_loss: 1.3329 - val_acc: 0.8380\n",
      "Epoch 20/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3329 - acc: 0.8365 - val_loss: 1.3346 - val_acc: 0.8446\n",
      "Epoch 21/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3335 - acc: 0.8365 - val_loss: 1.3319 - val_acc: 0.8402\n",
      "Epoch 22/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3331 - acc: 0.8393 - val_loss: 1.3332 - val_acc: 0.8368\n",
      "Epoch 23/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3333 - acc: 0.8374 - val_loss: 1.3349 - val_acc: 0.8310\n",
      "Epoch 24/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3334 - acc: 0.8391 - val_loss: 1.3335 - val_acc: 0.8408\n",
      "Epoch 25/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3326 - acc: 0.8375 - val_loss: 1.3306 - val_acc: 0.8288\n",
      "Epoch 26/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3337 - acc: 0.8358 - val_loss: 1.3348 - val_acc: 0.8336\n",
      "Epoch 27/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3331 - acc: 0.8373 - val_loss: 1.3355 - val_acc: 0.8352\n",
      "Epoch 28/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3330 - acc: 0.8374 - val_loss: 1.3310 - val_acc: 0.8300\n",
      "Epoch 29/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3329 - acc: 0.8382 - val_loss: 1.3380 - val_acc: 0.8392\n",
      "Epoch 30/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 1.3330 - acc: 0.8375 - val_loss: 1.3390 - val_acc: 0.8349\n",
      "Epoch 31/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3331 - acc: 0.8378 - val_loss: 1.3340 - val_acc: 0.8410\n",
      "Epoch 32/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3334 - acc: 0.8386 - val_loss: 1.3324 - val_acc: 0.8521\n",
      "Epoch 33/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3331 - acc: 0.8385 - val_loss: 1.3365 - val_acc: 0.8368\n",
      "Epoch 34/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3332 - acc: 0.8379 - val_loss: 1.3316 - val_acc: 0.8351\n",
      "Epoch 35/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3326 - acc: 0.8378 - val_loss: 1.3354 - val_acc: 0.8208\n",
      "Epoch 36/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3330 - acc: 0.8372 - val_loss: 1.3385 - val_acc: 0.8375\n",
      "Epoch 37/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3333 - acc: 0.8387 - val_loss: 1.3318 - val_acc: 0.8375\n",
      "Epoch 38/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3329 - acc: 0.8378 - val_loss: 1.3352 - val_acc: 0.8415\n",
      "Epoch 39/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3333 - acc: 0.8378 - val_loss: 1.3345 - val_acc: 0.8460\n",
      "Epoch 40/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3328 - acc: 0.8378 - val_loss: 1.3321 - val_acc: 0.8433\n",
      "Epoch 41/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3329 - acc: 0.8378 - val_loss: 1.3354 - val_acc: 0.8287\n",
      "Epoch 42/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 1.3327 - acc: 0.8386 - val_loss: 1.3333 - val_acc: 0.8324\n",
      "Epoch 43/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3328 - acc: 0.8379 - val_loss: 1.3348 - val_acc: 0.8408\n",
      "Epoch 44/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3331 - acc: 0.8373 - val_loss: 1.3368 - val_acc: 0.8280\n",
      "Epoch 45/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3329 - acc: 0.8369 - val_loss: 1.3321 - val_acc: 0.8439\n",
      "Epoch 46/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3328 - acc: 0.8376 - val_loss: 1.3330 - val_acc: 0.8381\n",
      "Epoch 47/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3330 - acc: 0.8368 - val_loss: 1.3315 - val_acc: 0.8471\n",
      "Epoch 48/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3328 - acc: 0.8394 - val_loss: 1.3362 - val_acc: 0.8454\n",
      "Epoch 49/150\n",
      "33600/33600 [==============================] - 3s 97us/step - loss: 1.3333 - acc: 0.8376 - val_loss: 1.3350 - val_acc: 0.8275\n",
      "Epoch 50/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3336 - acc: 0.8372 - val_loss: 1.3346 - val_acc: 0.8449\n",
      "Epoch 51/150\n",
      "33600/33600 [==============================] - 3s 93us/step - loss: 1.3333 - acc: 0.8373 - val_loss: 1.3333 - val_acc: 0.8383\n",
      "Epoch 52/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 1.3334 - acc: 0.8382 - val_loss: 1.3330 - val_acc: 0.8375\n",
      "Epoch 53/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3334 - acc: 0.8373 - val_loss: 1.3322 - val_acc: 0.8373\n",
      "Epoch 54/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3334 - acc: 0.8363 - val_loss: 1.3328 - val_acc: 0.8480\n",
      "Epoch 55/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3330 - acc: 0.8378 - val_loss: 1.3291 - val_acc: 0.8446\n",
      "Epoch 56/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3335 - acc: 0.8367 - val_loss: 1.3336 - val_acc: 0.8463\n",
      "Epoch 57/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3329 - acc: 0.8388 - val_loss: 1.3295 - val_acc: 0.8437\n",
      "Epoch 58/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3324 - acc: 0.8370 - val_loss: 1.3321 - val_acc: 0.8392\n",
      "Epoch 59/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3329 - acc: 0.8377 - val_loss: 1.3340 - val_acc: 0.8404\n",
      "Epoch 60/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3330 - acc: 0.8370 - val_loss: 1.3336 - val_acc: 0.8425\n",
      "Epoch 61/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3332 - acc: 0.8375 - val_loss: 1.3311 - val_acc: 0.8394\n",
      "Epoch 62/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3334 - acc: 0.8376 - val_loss: 1.3312 - val_acc: 0.8361\n",
      "Epoch 63/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3331 - acc: 0.8367 - val_loss: 1.3313 - val_acc: 0.8363\n",
      "Epoch 64/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3334 - acc: 0.8379 - val_loss: 1.3392 - val_acc: 0.8340\n",
      "Epoch 65/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3332 - acc: 0.8382 - val_loss: 1.3293 - val_acc: 0.8327\n",
      "Epoch 66/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3335 - acc: 0.8374 - val_loss: 1.3379 - val_acc: 0.8373\n",
      "Epoch 67/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3334 - acc: 0.8373 - val_loss: 1.3341 - val_acc: 0.8354\n",
      "Epoch 68/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3332 - acc: 0.8372 - val_loss: 1.3378 - val_acc: 0.8333\n",
      "Epoch 69/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3333 - acc: 0.8375 - val_loss: 1.3329 - val_acc: 0.8376\n",
      "Epoch 70/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3335 - acc: 0.8383 - val_loss: 1.3346 - val_acc: 0.8383\n",
      "Epoch 71/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3328 - acc: 0.8375 - val_loss: 1.3335 - val_acc: 0.8320\n",
      "Epoch 72/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8382 - val_loss: 1.3401 - val_acc: 0.8369\n",
      "Epoch 73/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3330 - acc: 0.8375 - val_loss: 1.3377 - val_acc: 0.8338\n",
      "Epoch 74/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3329 - acc: 0.8381 - val_loss: 1.3364 - val_acc: 0.8331\n",
      "Epoch 75/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3327 - acc: 0.8391 - val_loss: 1.3323 - val_acc: 0.8405\n",
      "Epoch 76/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3331 - acc: 0.8378 - val_loss: 1.3297 - val_acc: 0.8371\n",
      "Epoch 77/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3331 - acc: 0.8372 - val_loss: 1.3302 - val_acc: 0.8385\n",
      "Epoch 78/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3329 - acc: 0.8367 - val_loss: 1.3308 - val_acc: 0.8412\n",
      "Epoch 79/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3327 - acc: 0.8383 - val_loss: 1.3356 - val_acc: 0.8313\n",
      "Epoch 80/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3332 - acc: 0.8372 - val_loss: 1.3306 - val_acc: 0.8444\n",
      "Epoch 81/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3333 - acc: 0.8355 - val_loss: 1.3325 - val_acc: 0.8307\n",
      "Epoch 82/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 1.3334 - acc: 0.8385 - val_loss: 1.3303 - val_acc: 0.8368\n",
      "Epoch 83/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3328 - acc: 0.8377 - val_loss: 1.3346 - val_acc: 0.8417\n",
      "Epoch 84/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3332 - acc: 0.8371 - val_loss: 1.3364 - val_acc: 0.8392\n",
      "Epoch 85/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3332 - acc: 0.8383 - val_loss: 1.3324 - val_acc: 0.8346\n",
      "Epoch 86/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3334 - acc: 0.8379 - val_loss: 1.3308 - val_acc: 0.8327\n",
      "Epoch 87/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3334 - acc: 0.8363 - val_loss: 1.3328 - val_acc: 0.8370\n",
      "Epoch 88/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3334 - acc: 0.8386 - val_loss: 1.3313 - val_acc: 0.8375\n",
      "Epoch 89/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3332 - acc: 0.8375 - val_loss: 1.3326 - val_acc: 0.8380\n",
      "Epoch 90/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3324 - acc: 0.8377 - val_loss: 1.3317 - val_acc: 0.8411\n",
      "Epoch 91/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3335 - acc: 0.8383 - val_loss: 1.3329 - val_acc: 0.8462\n",
      "Epoch 92/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3333 - acc: 0.8375 - val_loss: 1.3296 - val_acc: 0.8354\n",
      "Epoch 93/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3331 - acc: 0.8371 - val_loss: 1.3336 - val_acc: 0.8412\n",
      "Epoch 94/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3324 - acc: 0.8384 - val_loss: 1.3310 - val_acc: 0.8412\n",
      "Epoch 95/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3330 - acc: 0.8368 - val_loss: 1.3355 - val_acc: 0.8452\n",
      "Epoch 96/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3333 - acc: 0.8381 - val_loss: 1.3317 - val_acc: 0.8363\n",
      "Epoch 97/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3328 - acc: 0.8372 - val_loss: 1.3364 - val_acc: 0.8308\n",
      "Epoch 98/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8370 - val_loss: 1.3335 - val_acc: 0.8427\n",
      "Epoch 99/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3333 - acc: 0.8381 - val_loss: 1.3324 - val_acc: 0.8269\n",
      "Epoch 100/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3331 - acc: 0.8382 - val_loss: 1.3310 - val_acc: 0.8355\n",
      "Epoch 101/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3331 - acc: 0.8385 - val_loss: 1.3324 - val_acc: 0.8280\n",
      "Epoch 102/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8374 - val_loss: 1.3343 - val_acc: 0.8375\n",
      "Epoch 103/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3331 - acc: 0.8380 - val_loss: 1.3314 - val_acc: 0.8270\n",
      "Epoch 104/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 1.3336 - acc: 0.8372 - val_loss: 1.3334 - val_acc: 0.8321\n",
      "Epoch 105/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3333 - acc: 0.8371 - val_loss: 1.3308 - val_acc: 0.8418\n",
      "Epoch 106/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3331 - acc: 0.8370 - val_loss: 1.3311 - val_acc: 0.8319\n",
      "Epoch 107/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3332 - acc: 0.8371 - val_loss: 1.3308 - val_acc: 0.8412\n",
      "Epoch 108/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3328 - acc: 0.8375 - val_loss: 1.3371 - val_acc: 0.8381\n",
      "Epoch 109/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3335 - acc: 0.8354 - val_loss: 1.3302 - val_acc: 0.8368\n",
      "Epoch 110/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3328 - acc: 0.8378 - val_loss: 1.3309 - val_acc: 0.8343\n",
      "Epoch 111/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3330 - acc: 0.8371 - val_loss: 1.3379 - val_acc: 0.8430\n",
      "Epoch 112/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3332 - acc: 0.8376 - val_loss: 1.3344 - val_acc: 0.8486\n",
      "Epoch 113/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3328 - acc: 0.8390 - val_loss: 1.3334 - val_acc: 0.8369\n",
      "Epoch 114/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3336 - acc: 0.8376 - val_loss: 1.3326 - val_acc: 0.8333\n",
      "Epoch 115/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3330 - acc: 0.8376 - val_loss: 1.3364 - val_acc: 0.8420\n",
      "Epoch 116/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3335 - acc: 0.8370 - val_loss: 1.3308 - val_acc: 0.8385\n",
      "Epoch 117/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3330 - acc: 0.8365 - val_loss: 1.3414 - val_acc: 0.8349\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3326 - acc: 0.8363 - val_loss: 1.3314 - val_acc: 0.8404\n",
      "Epoch 119/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3337 - acc: 0.8361 - val_loss: 1.3327 - val_acc: 0.8371\n",
      "Epoch 120/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 1.3330 - acc: 0.8361 - val_loss: 1.3306 - val_acc: 0.8437\n",
      "Epoch 121/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 1.3332 - acc: 0.8365 - val_loss: 1.3299 - val_acc: 0.8361\n",
      "Epoch 122/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8366 - val_loss: 1.3323 - val_acc: 0.8321\n",
      "Epoch 123/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3333 - acc: 0.8381 - val_loss: 1.3316 - val_acc: 0.8438\n",
      "Epoch 124/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3329 - acc: 0.8381 - val_loss: 1.3328 - val_acc: 0.8383\n",
      "Epoch 125/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3331 - acc: 0.8377 - val_loss: 1.3339 - val_acc: 0.8402\n",
      "Epoch 126/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3328 - acc: 0.8378 - val_loss: 1.3426 - val_acc: 0.8210\n",
      "Epoch 127/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3333 - acc: 0.8394 - val_loss: 1.3309 - val_acc: 0.8424\n",
      "Epoch 128/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3334 - acc: 0.8361 - val_loss: 1.3306 - val_acc: 0.8438\n",
      "Epoch 129/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3332 - acc: 0.8372 - val_loss: 1.3318 - val_acc: 0.8376\n",
      "Epoch 130/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3333 - acc: 0.8382 - val_loss: 1.3339 - val_acc: 0.8400\n",
      "Epoch 131/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3334 - acc: 0.8367 - val_loss: 1.3291 - val_acc: 0.8396\n",
      "Epoch 132/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3328 - acc: 0.8371 - val_loss: 1.3315 - val_acc: 0.8375\n",
      "Epoch 133/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3334 - acc: 0.8380 - val_loss: 1.3360 - val_acc: 0.8317\n",
      "Epoch 134/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3331 - acc: 0.8392 - val_loss: 1.3307 - val_acc: 0.8381\n",
      "Epoch 135/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 1.3335 - acc: 0.8381 - val_loss: 1.3289 - val_acc: 0.8432\n",
      "Epoch 136/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3329 - acc: 0.8364 - val_loss: 1.3338 - val_acc: 0.8412\n",
      "Epoch 137/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3333 - acc: 0.8385 - val_loss: 1.3330 - val_acc: 0.8464\n",
      "Epoch 138/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8355 - val_loss: 1.3359 - val_acc: 0.8377\n",
      "Epoch 139/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3330 - acc: 0.8377 - val_loss: 1.3311 - val_acc: 0.8440\n",
      "Epoch 140/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3328 - acc: 0.8379 - val_loss: 1.3328 - val_acc: 0.8358\n",
      "Epoch 141/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3338 - acc: 0.8382 - val_loss: 1.3309 - val_acc: 0.8439\n",
      "Epoch 142/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8374 - val_loss: 1.3373 - val_acc: 0.8444\n",
      "Epoch 143/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8379 - val_loss: 1.3353 - val_acc: 0.8354\n",
      "Epoch 144/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3329 - acc: 0.8378 - val_loss: 1.3351 - val_acc: 0.8388\n",
      "Epoch 145/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 1.3334 - acc: 0.8360 - val_loss: 1.3339 - val_acc: 0.8425\n",
      "Epoch 146/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3331 - acc: 0.8365 - val_loss: 1.3307 - val_acc: 0.8343\n",
      "Epoch 147/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 1.3329 - acc: 0.8373 - val_loss: 1.3330 - val_acc: 0.8258\n",
      "Epoch 148/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3331 - acc: 0.8377 - val_loss: 1.3354 - val_acc: 0.8345\n",
      "Epoch 149/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 1.3330 - acc: 0.8378 - val_loss: 1.3390 - val_acc: 0.8233\n",
      "Epoch 150/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 1.3332 - acc: 0.8377 - val_loss: 1.3332 - val_acc: 0.8370\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/150\n",
      "33600/33600 [==============================] - 3s 103us/step - loss: 0.8762 - acc: 0.8389 - val_loss: 0.6969 - val_acc: 0.8806\n",
      "Epoch 2/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.6845 - acc: 0.8878 - val_loss: 0.6831 - val_acc: 0.8839\n",
      "Epoch 3/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6771 - acc: 0.8889 - val_loss: 0.6813 - val_acc: 0.8886\n",
      "Epoch 4/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6744 - acc: 0.8907 - val_loss: 0.6791 - val_acc: 0.8887\n",
      "Epoch 5/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6731 - acc: 0.8898 - val_loss: 0.6768 - val_acc: 0.8876\n",
      "Epoch 6/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6715 - acc: 0.8906 - val_loss: 0.6783 - val_acc: 0.8860\n",
      "Epoch 7/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6705 - acc: 0.8911 - val_loss: 0.6800 - val_acc: 0.8840\n",
      "Epoch 8/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6698 - acc: 0.8908 - val_loss: 0.6730 - val_acc: 0.8849\n",
      "Epoch 9/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6689 - acc: 0.8914 - val_loss: 0.6715 - val_acc: 0.8867\n",
      "Epoch 10/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6684 - acc: 0.8923 - val_loss: 0.6754 - val_acc: 0.8880\n",
      "Epoch 11/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6680 - acc: 0.8910 - val_loss: 0.6744 - val_acc: 0.8914\n",
      "Epoch 12/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6677 - acc: 0.8909 - val_loss: 0.6730 - val_acc: 0.8896\n",
      "Epoch 13/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6670 - acc: 0.8914 - val_loss: 0.6739 - val_acc: 0.8836\n",
      "Epoch 14/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6672 - acc: 0.8905 - val_loss: 0.6711 - val_acc: 0.8882\n",
      "Epoch 15/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6669 - acc: 0.8907 - val_loss: 0.6730 - val_acc: 0.8869\n",
      "Epoch 16/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6665 - acc: 0.8909 - val_loss: 0.6715 - val_acc: 0.8894\n",
      "Epoch 17/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6663 - acc: 0.8915 - val_loss: 0.6702 - val_acc: 0.8881\n",
      "Epoch 18/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6660 - acc: 0.8923 - val_loss: 0.6730 - val_acc: 0.8832\n",
      "Epoch 19/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.6663 - acc: 0.8906 - val_loss: 0.6716 - val_acc: 0.8829\n",
      "Epoch 20/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.6662 - acc: 0.8906 - val_loss: 0.6722 - val_acc: 0.8851\n",
      "Epoch 21/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6656 - acc: 0.8916 - val_loss: 0.6712 - val_acc: 0.8876\n",
      "Epoch 22/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6660 - acc: 0.8907 - val_loss: 0.6693 - val_acc: 0.8858\n",
      "Epoch 23/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6660 - acc: 0.8909 - val_loss: 0.6734 - val_acc: 0.8852\n",
      "Epoch 24/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6659 - acc: 0.8910 - val_loss: 0.6702 - val_acc: 0.8860\n",
      "Epoch 25/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6657 - acc: 0.8912 - val_loss: 0.6712 - val_acc: 0.8855\n",
      "Epoch 26/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6659 - acc: 0.8910 - val_loss: 0.6744 - val_acc: 0.8839\n",
      "Epoch 27/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6654 - acc: 0.8910 - val_loss: 0.6684 - val_acc: 0.8868\n",
      "Epoch 28/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6653 - acc: 0.8910 - val_loss: 0.6718 - val_acc: 0.8849\n",
      "Epoch 29/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6659 - acc: 0.8905 - val_loss: 0.6710 - val_acc: 0.8887\n",
      "Epoch 30/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6654 - acc: 0.8899 - val_loss: 0.6731 - val_acc: 0.8858\n",
      "Epoch 31/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6651 - acc: 0.8907 - val_loss: 0.6696 - val_acc: 0.8879\n",
      "Epoch 32/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6653 - acc: 0.8909 - val_loss: 0.6734 - val_acc: 0.8830\n",
      "Epoch 33/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6650 - acc: 0.8904 - val_loss: 0.6741 - val_acc: 0.8814\n",
      "Epoch 34/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6651 - acc: 0.8910 - val_loss: 0.6758 - val_acc: 0.8815\n",
      "Epoch 35/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6654 - acc: 0.8906 - val_loss: 0.6744 - val_acc: 0.8820\n",
      "Epoch 36/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6653 - acc: 0.8909 - val_loss: 0.6704 - val_acc: 0.8844\n",
      "Epoch 37/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6651 - acc: 0.8899 - val_loss: 0.6711 - val_acc: 0.8840\n",
      "Epoch 38/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6656 - acc: 0.8900 - val_loss: 0.6713 - val_acc: 0.8876\n",
      "Epoch 39/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6651 - acc: 0.8899 - val_loss: 0.6739 - val_acc: 0.8858\n",
      "Epoch 40/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6654 - acc: 0.8906 - val_loss: 0.6692 - val_acc: 0.8862\n",
      "Epoch 41/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6660 - acc: 0.8908 - val_loss: 0.6730 - val_acc: 0.8840\n",
      "Epoch 42/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6656 - acc: 0.8903 - val_loss: 0.6734 - val_acc: 0.8863\n",
      "Epoch 43/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6655 - acc: 0.8899 - val_loss: 0.6701 - val_acc: 0.8865\n",
      "Epoch 44/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6659 - acc: 0.8902 - val_loss: 0.6704 - val_acc: 0.8857\n",
      "Epoch 45/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6655 - acc: 0.8912 - val_loss: 0.6698 - val_acc: 0.8846\n",
      "Epoch 46/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6657 - acc: 0.8899 - val_loss: 0.6688 - val_acc: 0.8870\n",
      "Epoch 47/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6655 - acc: 0.8896 - val_loss: 0.6711 - val_acc: 0.8870\n",
      "Epoch 48/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6652 - acc: 0.8900 - val_loss: 0.6706 - val_acc: 0.8876\n",
      "Epoch 49/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6656 - acc: 0.8901 - val_loss: 0.6686 - val_acc: 0.8848\n",
      "Epoch 50/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6652 - acc: 0.8907 - val_loss: 0.6690 - val_acc: 0.8861\n",
      "Epoch 51/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6656 - acc: 0.8909 - val_loss: 0.6719 - val_acc: 0.8821\n",
      "Epoch 52/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6657 - acc: 0.8905 - val_loss: 0.6726 - val_acc: 0.8824\n",
      "Epoch 53/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.6653 - acc: 0.8898 - val_loss: 0.6688 - val_acc: 0.8856\n",
      "Epoch 54/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6650 - acc: 0.8903 - val_loss: 0.6723 - val_acc: 0.8857\n",
      "Epoch 55/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6650 - acc: 0.8910 - val_loss: 0.6705 - val_acc: 0.8863\n",
      "Epoch 56/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6655 - acc: 0.8903 - val_loss: 0.6711 - val_acc: 0.8861\n",
      "Epoch 57/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6654 - acc: 0.8898 - val_loss: 0.6720 - val_acc: 0.8837\n",
      "Epoch 58/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6655 - acc: 0.8902 - val_loss: 0.6693 - val_acc: 0.8855\n",
      "Epoch 59/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6655 - acc: 0.8898 - val_loss: 0.6722 - val_acc: 0.8833\n",
      "Epoch 60/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6654 - acc: 0.8904 - val_loss: 0.6719 - val_acc: 0.8854\n",
      "Epoch 61/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6657 - acc: 0.8900 - val_loss: 0.6709 - val_acc: 0.8849\n",
      "Epoch 62/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6654 - acc: 0.8901 - val_loss: 0.6718 - val_acc: 0.8840\n",
      "Epoch 63/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6654 - acc: 0.8895 - val_loss: 0.6689 - val_acc: 0.8875\n",
      "Epoch 64/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6649 - acc: 0.8894 - val_loss: 0.6727 - val_acc: 0.8865\n",
      "Epoch 65/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6652 - acc: 0.8899 - val_loss: 0.6735 - val_acc: 0.8855\n",
      "Epoch 66/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6659 - acc: 0.8897 - val_loss: 0.6701 - val_acc: 0.8882\n",
      "Epoch 67/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6656 - acc: 0.8916 - val_loss: 0.6701 - val_acc: 0.8856\n",
      "Epoch 68/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6652 - acc: 0.8896 - val_loss: 0.6743 - val_acc: 0.8792\n",
      "Epoch 69/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6656 - acc: 0.8897 - val_loss: 0.6747 - val_acc: 0.8845\n",
      "Epoch 70/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6654 - acc: 0.8905 - val_loss: 0.6720 - val_acc: 0.8840\n",
      "Epoch 71/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6653 - acc: 0.8893 - val_loss: 0.6753 - val_acc: 0.8843\n",
      "Epoch 72/150\n",
      "33600/33600 [==============================] - 3s 93us/step - loss: 0.6658 - acc: 0.8908 - val_loss: 0.6722 - val_acc: 0.8873\n",
      "Epoch 73/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 0.6659 - acc: 0.8906 - val_loss: 0.6711 - val_acc: 0.8850\n",
      "Epoch 74/150\n",
      "33600/33600 [==============================] - 3s 96us/step - loss: 0.6657 - acc: 0.8886 - val_loss: 0.6694 - val_acc: 0.8876\n",
      "Epoch 75/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.6656 - acc: 0.8893 - val_loss: 0.6759 - val_acc: 0.8856\n",
      "Epoch 76/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6657 - acc: 0.8892 - val_loss: 0.6713 - val_acc: 0.8863\n",
      "Epoch 77/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6653 - acc: 0.8905 - val_loss: 0.6708 - val_acc: 0.8857\n",
      "Epoch 78/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6657 - acc: 0.8897 - val_loss: 0.6754 - val_acc: 0.8838\n",
      "Epoch 79/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6655 - acc: 0.8895 - val_loss: 0.6718 - val_acc: 0.8865\n",
      "Epoch 80/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6652 - acc: 0.8904 - val_loss: 0.6700 - val_acc: 0.8860\n",
      "Epoch 81/150\n",
      "33600/33600 [==============================] - 3s 93us/step - loss: 0.6656 - acc: 0.8891 - val_loss: 0.6694 - val_acc: 0.8869\n",
      "Epoch 82/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6653 - acc: 0.8908 - val_loss: 0.6693 - val_acc: 0.8854\n",
      "Epoch 83/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6651 - acc: 0.8915 - val_loss: 0.6709 - val_acc: 0.8882\n",
      "Epoch 84/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6656 - acc: 0.8891 - val_loss: 0.6698 - val_acc: 0.8867\n",
      "Epoch 85/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6653 - acc: 0.8905 - val_loss: 0.6684 - val_acc: 0.8850\n",
      "Epoch 86/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6654 - acc: 0.8897 - val_loss: 0.6715 - val_acc: 0.8860\n",
      "Epoch 87/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6656 - acc: 0.8898 - val_loss: 0.6701 - val_acc: 0.8871\n",
      "Epoch 88/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6652 - acc: 0.8899 - val_loss: 0.6712 - val_acc: 0.8862\n",
      "Epoch 89/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6653 - acc: 0.8900 - val_loss: 0.6729 - val_acc: 0.8867\n",
      "Epoch 90/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6653 - acc: 0.8908 - val_loss: 0.6699 - val_acc: 0.8874\n",
      "Epoch 91/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.6655 - acc: 0.8901 - val_loss: 0.6718 - val_acc: 0.8862\n",
      "Epoch 92/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6654 - acc: 0.8904 - val_loss: 0.6735 - val_acc: 0.8830\n",
      "Epoch 93/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6655 - acc: 0.8899 - val_loss: 0.6687 - val_acc: 0.8865\n",
      "Epoch 94/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6652 - acc: 0.8903 - val_loss: 0.6693 - val_acc: 0.8849\n",
      "Epoch 95/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6652 - acc: 0.8895 - val_loss: 0.6701 - val_acc: 0.8860\n",
      "Epoch 96/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6650 - acc: 0.8902 - val_loss: 0.6707 - val_acc: 0.8849\n",
      "Epoch 97/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6656 - acc: 0.8902 - val_loss: 0.6694 - val_acc: 0.8855\n",
      "Epoch 98/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6657 - acc: 0.8909 - val_loss: 0.6702 - val_acc: 0.8863\n",
      "Epoch 99/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6656 - acc: 0.8897 - val_loss: 0.6726 - val_acc: 0.8836\n",
      "Epoch 100/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6655 - acc: 0.8899 - val_loss: 0.6728 - val_acc: 0.8860\n",
      "Epoch 101/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6650 - acc: 0.8899 - val_loss: 0.6712 - val_acc: 0.8843\n",
      "Epoch 102/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6652 - acc: 0.8898 - val_loss: 0.6723 - val_acc: 0.8855\n",
      "Epoch 103/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6653 - acc: 0.8905 - val_loss: 0.6682 - val_acc: 0.8875\n",
      "Epoch 104/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6653 - acc: 0.8905 - val_loss: 0.6729 - val_acc: 0.8862\n",
      "Epoch 105/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6656 - acc: 0.8901 - val_loss: 0.6726 - val_acc: 0.8858\n",
      "Epoch 106/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6653 - acc: 0.8901 - val_loss: 0.6704 - val_acc: 0.8874\n",
      "Epoch 107/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6655 - acc: 0.8907 - val_loss: 0.6695 - val_acc: 0.8863\n",
      "Epoch 108/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6655 - acc: 0.8898 - val_loss: 0.6708 - val_acc: 0.8850\n",
      "Epoch 109/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6656 - acc: 0.8896 - val_loss: 0.6724 - val_acc: 0.8876\n",
      "Epoch 110/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6654 - acc: 0.8900 - val_loss: 0.6712 - val_acc: 0.8863\n",
      "Epoch 111/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6653 - acc: 0.8901 - val_loss: 0.6786 - val_acc: 0.8814\n",
      "Epoch 112/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6658 - acc: 0.8912 - val_loss: 0.6698 - val_acc: 0.8881\n",
      "Epoch 113/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6656 - acc: 0.8902 - val_loss: 0.6711 - val_acc: 0.8832\n",
      "Epoch 114/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6656 - acc: 0.8905 - val_loss: 0.6699 - val_acc: 0.8875\n",
      "Epoch 115/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.6653 - acc: 0.8901 - val_loss: 0.6698 - val_acc: 0.8840\n",
      "Epoch 116/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6654 - acc: 0.8897 - val_loss: 0.6749 - val_acc: 0.8831\n",
      "Epoch 117/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6654 - acc: 0.8909 - val_loss: 0.6710 - val_acc: 0.8854\n",
      "Epoch 118/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6653 - acc: 0.8901 - val_loss: 0.6703 - val_acc: 0.8823\n",
      "Epoch 119/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6651 - acc: 0.8905 - val_loss: 0.6717 - val_acc: 0.8851\n",
      "Epoch 120/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6653 - acc: 0.8896 - val_loss: 0.6694 - val_acc: 0.8875\n",
      "Epoch 121/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6653 - acc: 0.8911 - val_loss: 0.6705 - val_acc: 0.8875\n",
      "Epoch 122/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6650 - acc: 0.8902 - val_loss: 0.6724 - val_acc: 0.8867\n",
      "Epoch 123/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6652 - acc: 0.8894 - val_loss: 0.6717 - val_acc: 0.8856\n",
      "Epoch 124/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6656 - acc: 0.8892 - val_loss: 0.6713 - val_acc: 0.8873\n",
      "Epoch 125/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.6653 - acc: 0.8902 - val_loss: 0.6714 - val_acc: 0.8849\n",
      "Epoch 126/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6651 - acc: 0.8911 - val_loss: 0.6708 - val_acc: 0.8843\n",
      "Epoch 127/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6656 - acc: 0.8906 - val_loss: 0.6695 - val_acc: 0.8869\n",
      "Epoch 128/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6649 - acc: 0.8905 - val_loss: 0.6713 - val_acc: 0.8845\n",
      "Epoch 129/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6654 - acc: 0.8903 - val_loss: 0.6702 - val_acc: 0.8875\n",
      "Epoch 130/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6655 - acc: 0.8905 - val_loss: 0.6705 - val_acc: 0.8862\n",
      "Epoch 131/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6654 - acc: 0.8904 - val_loss: 0.6712 - val_acc: 0.8857\n",
      "Epoch 132/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6649 - acc: 0.8895 - val_loss: 0.6716 - val_acc: 0.8850\n",
      "Epoch 133/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6655 - acc: 0.8902 - val_loss: 0.6698 - val_acc: 0.8873\n",
      "Epoch 134/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6652 - acc: 0.8904 - val_loss: 0.6732 - val_acc: 0.8855\n",
      "Epoch 135/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.6652 - acc: 0.8904 - val_loss: 0.6693 - val_acc: 0.8844\n",
      "Epoch 136/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.6654 - acc: 0.8901 - val_loss: 0.6757 - val_acc: 0.8836\n",
      "Epoch 137/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6657 - acc: 0.8897 - val_loss: 0.6710 - val_acc: 0.8869\n",
      "Epoch 138/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6654 - acc: 0.8899 - val_loss: 0.6709 - val_acc: 0.8862\n",
      "Epoch 139/150\n",
      "33600/33600 [==============================] - 3s 96us/step - loss: 0.6657 - acc: 0.8888 - val_loss: 0.6699 - val_acc: 0.8868\n",
      "Epoch 140/150\n",
      "33600/33600 [==============================] - 3s 93us/step - loss: 0.6654 - acc: 0.8908 - val_loss: 0.6688 - val_acc: 0.8860\n",
      "Epoch 141/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.6657 - acc: 0.8904 - val_loss: 0.6731 - val_acc: 0.8838\n",
      "Epoch 142/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6653 - acc: 0.8902 - val_loss: 0.6697 - val_acc: 0.8870\n",
      "Epoch 143/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6655 - acc: 0.8895 - val_loss: 0.6696 - val_acc: 0.8876\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 144/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6656 - acc: 0.8895 - val_loss: 0.6725 - val_acc: 0.8817\n",
      "Epoch 145/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6657 - acc: 0.8900 - val_loss: 0.6707 - val_acc: 0.8831\n",
      "Epoch 146/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.6653 - acc: 0.8900 - val_loss: 0.6692 - val_acc: 0.8885\n",
      "Epoch 147/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.6654 - acc: 0.8901 - val_loss: 0.6709 - val_acc: 0.8880\n",
      "Epoch 148/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.6648 - acc: 0.8896 - val_loss: 0.6720 - val_acc: 0.8875\n",
      "Epoch 149/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.6652 - acc: 0.8896 - val_loss: 0.6730 - val_acc: 0.8875\n",
      "Epoch 150/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.6658 - acc: 0.8893 - val_loss: 0.6698 - val_acc: 0.8875\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/150\n",
      "33600/33600 [==============================] - 4s 121us/step - loss: 0.6631 - acc: 0.8437 - val_loss: 0.4433 - val_acc: 0.8950\n",
      "Epoch 2/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.4195 - acc: 0.9021 - val_loss: 0.4164 - val_acc: 0.9014\n",
      "Epoch 3/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.4026 - acc: 0.9082 - val_loss: 0.4096 - val_acc: 0.9073\n",
      "Epoch 4/150\n",
      "33600/33600 [==============================] - 3s 95us/step - loss: 0.3964 - acc: 0.9114 - val_loss: 0.4088 - val_acc: 0.9062\n",
      "Epoch 5/150\n",
      "33600/33600 [==============================] - 3s 95us/step - loss: 0.3937 - acc: 0.9128 - val_loss: 0.4083 - val_acc: 0.9080\n",
      "Epoch 6/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3914 - acc: 0.9134 - val_loss: 0.4054 - val_acc: 0.9092\n",
      "Epoch 7/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3909 - acc: 0.9147 - val_loss: 0.4062 - val_acc: 0.9090\n",
      "Epoch 8/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3904 - acc: 0.9148 - val_loss: 0.4048 - val_acc: 0.9106\n",
      "Epoch 9/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3896 - acc: 0.9156 - val_loss: 0.4062 - val_acc: 0.9095\n",
      "Epoch 10/150\n",
      "33600/33600 [==============================] - 3s 95us/step - loss: 0.3897 - acc: 0.9162 - val_loss: 0.4052 - val_acc: 0.9092\n",
      "Epoch 11/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3890 - acc: 0.9144 - val_loss: 0.4062 - val_acc: 0.9106\n",
      "Epoch 12/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3891 - acc: 0.9153 - val_loss: 0.4054 - val_acc: 0.9117\n",
      "Epoch 13/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3883 - acc: 0.9158 - val_loss: 0.4065 - val_acc: 0.9105\n",
      "Epoch 14/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3886 - acc: 0.9160 - val_loss: 0.4057 - val_acc: 0.9110\n",
      "Epoch 15/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3888 - acc: 0.9165 - val_loss: 0.4074 - val_acc: 0.9104\n",
      "Epoch 16/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3883 - acc: 0.9164 - val_loss: 0.4042 - val_acc: 0.9102\n",
      "Epoch 17/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3882 - acc: 0.9155 - val_loss: 0.4055 - val_acc: 0.9107\n",
      "Epoch 18/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3879 - acc: 0.9166 - val_loss: 0.4055 - val_acc: 0.9093\n",
      "Epoch 19/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3879 - acc: 0.9164 - val_loss: 0.4042 - val_acc: 0.9113\n",
      "Epoch 20/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3879 - acc: 0.9152 - val_loss: 0.4064 - val_acc: 0.9101\n",
      "Epoch 21/150\n",
      "33600/33600 [==============================] - 3s 99us/step - loss: 0.3877 - acc: 0.9168 - val_loss: 0.4062 - val_acc: 0.9101\n",
      "Epoch 22/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 0.3880 - acc: 0.9161 - val_loss: 0.4051 - val_acc: 0.9108\n",
      "Epoch 23/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3875 - acc: 0.9170 - val_loss: 0.4064 - val_acc: 0.9095\n",
      "Epoch 24/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3876 - acc: 0.9160 - val_loss: 0.4043 - val_acc: 0.9127\n",
      "Epoch 25/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3879 - acc: 0.9166 - val_loss: 0.4059 - val_acc: 0.9107\n",
      "Epoch 26/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3876 - acc: 0.9158 - val_loss: 0.4072 - val_acc: 0.9092\n",
      "Epoch 27/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3875 - acc: 0.9163 - val_loss: 0.4065 - val_acc: 0.9098\n",
      "Epoch 28/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3878 - acc: 0.9162 - val_loss: 0.4077 - val_acc: 0.9101\n",
      "Epoch 29/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3875 - acc: 0.9156 - val_loss: 0.4068 - val_acc: 0.9107\n",
      "Epoch 30/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3871 - acc: 0.9168 - val_loss: 0.4060 - val_acc: 0.9121\n",
      "Epoch 31/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3877 - acc: 0.9159 - val_loss: 0.4046 - val_acc: 0.9102\n",
      "Epoch 32/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3872 - acc: 0.9161 - val_loss: 0.4075 - val_acc: 0.9102\n",
      "Epoch 33/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3876 - acc: 0.9157 - val_loss: 0.4064 - val_acc: 0.9105\n",
      "Epoch 34/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3873 - acc: 0.9159 - val_loss: 0.4060 - val_acc: 0.9114\n",
      "Epoch 35/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3872 - acc: 0.9156 - val_loss: 0.4047 - val_acc: 0.9090\n",
      "Epoch 36/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3876 - acc: 0.9160 - val_loss: 0.4070 - val_acc: 0.9070\n",
      "Epoch 37/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3874 - acc: 0.9162 - val_loss: 0.4057 - val_acc: 0.9106\n",
      "Epoch 38/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3874 - acc: 0.9157 - val_loss: 0.4038 - val_acc: 0.9108\n",
      "Epoch 39/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3870 - acc: 0.9154 - val_loss: 0.4056 - val_acc: 0.9123\n",
      "Epoch 40/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3876 - acc: 0.9163 - val_loss: 0.4067 - val_acc: 0.9102\n",
      "Epoch 41/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3873 - acc: 0.9165 - val_loss: 0.4039 - val_acc: 0.9124\n",
      "Epoch 42/150\n",
      "33600/33600 [==============================] - 3s 99us/step - loss: 0.3873 - acc: 0.9162 - val_loss: 0.4050 - val_acc: 0.9094\n",
      "Epoch 43/150\n",
      "33600/33600 [==============================] - 4s 105us/step - loss: 0.3868 - acc: 0.9163 - val_loss: 0.4043 - val_acc: 0.9081\n",
      "Epoch 44/150\n",
      "33600/33600 [==============================] - 3s 99us/step - loss: 0.3872 - acc: 0.9156 - val_loss: 0.4069 - val_acc: 0.9096\n",
      "Epoch 45/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3871 - acc: 0.9161 - val_loss: 0.4056 - val_acc: 0.9111\n",
      "Epoch 46/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3877 - acc: 0.9159 - val_loss: 0.4061 - val_acc: 0.9104\n",
      "Epoch 47/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3872 - acc: 0.9163 - val_loss: 0.4038 - val_acc: 0.9106\n",
      "Epoch 48/150\n",
      "33600/33600 [==============================] - 3s 99us/step - loss: 0.3873 - acc: 0.9166 - val_loss: 0.4073 - val_acc: 0.9096\n",
      "Epoch 49/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3875 - acc: 0.9156 - val_loss: 0.4055 - val_acc: 0.9087\n",
      "Epoch 50/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3875 - acc: 0.9160 - val_loss: 0.4047 - val_acc: 0.9107\n",
      "Epoch 51/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3876 - acc: 0.9168 - val_loss: 0.4072 - val_acc: 0.9098\n",
      "Epoch 52/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3871 - acc: 0.9169 - val_loss: 0.4068 - val_acc: 0.9102\n",
      "Epoch 53/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3872 - acc: 0.9154 - val_loss: 0.4039 - val_acc: 0.9115\n",
      "Epoch 54/150\n",
      "33600/33600 [==============================] - 3s 95us/step - loss: 0.3872 - acc: 0.9149 - val_loss: 0.4065 - val_acc: 0.9085\n",
      "Epoch 55/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3873 - acc: 0.9161 - val_loss: 0.4070 - val_acc: 0.9083\n",
      "Epoch 56/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3870 - acc: 0.9160 - val_loss: 0.4062 - val_acc: 0.9090\n",
      "Epoch 57/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3872 - acc: 0.9157 - val_loss: 0.4079 - val_acc: 0.9086\n",
      "Epoch 58/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3872 - acc: 0.9166 - val_loss: 0.4051 - val_acc: 0.9107\n",
      "Epoch 59/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3873 - acc: 0.9154 - val_loss: 0.4111 - val_acc: 0.9050\n",
      "Epoch 60/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3873 - acc: 0.9156 - val_loss: 0.4064 - val_acc: 0.9083\n",
      "Epoch 61/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3877 - acc: 0.9161 - val_loss: 0.4065 - val_acc: 0.9064\n",
      "Epoch 62/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 0.3870 - acc: 0.9160 - val_loss: 0.4103 - val_acc: 0.9093\n",
      "Epoch 63/150\n",
      "33600/33600 [==============================] - 3s 93us/step - loss: 0.3868 - acc: 0.9161 - val_loss: 0.4068 - val_acc: 0.9105\n",
      "Epoch 64/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3870 - acc: 0.9154 - val_loss: 0.4089 - val_acc: 0.9065\n",
      "Epoch 65/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3869 - acc: 0.9157 - val_loss: 0.4045 - val_acc: 0.9098\n",
      "Epoch 66/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3870 - acc: 0.9160 - val_loss: 0.4107 - val_acc: 0.9075\n",
      "Epoch 67/150\n",
      "33600/33600 [==============================] - 3s 95us/step - loss: 0.3872 - acc: 0.9158 - val_loss: 0.4031 - val_acc: 0.9110\n",
      "Epoch 68/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3865 - acc: 0.9154 - val_loss: 0.4032 - val_acc: 0.9124\n",
      "Epoch 69/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 0.3868 - acc: 0.9159 - val_loss: 0.4043 - val_acc: 0.9120\n",
      "Epoch 70/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3869 - acc: 0.9156 - val_loss: 0.4031 - val_acc: 0.9117\n",
      "Epoch 71/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3864 - acc: 0.9153 - val_loss: 0.4095 - val_acc: 0.9060\n",
      "Epoch 72/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3869 - acc: 0.9151 - val_loss: 0.4056 - val_acc: 0.9085\n",
      "Epoch 73/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3867 - acc: 0.9166 - val_loss: 0.4053 - val_acc: 0.9090\n",
      "Epoch 74/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3868 - acc: 0.9154 - val_loss: 0.4065 - val_acc: 0.9086\n",
      "Epoch 75/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3870 - acc: 0.9154 - val_loss: 0.4058 - val_acc: 0.9101\n",
      "Epoch 76/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3873 - acc: 0.9158 - val_loss: 0.4049 - val_acc: 0.9100\n",
      "Epoch 77/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3870 - acc: 0.9161 - val_loss: 0.4070 - val_acc: 0.9100\n",
      "Epoch 78/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3871 - acc: 0.9153 - val_loss: 0.4053 - val_acc: 0.9113\n",
      "Epoch 79/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3868 - acc: 0.9160 - val_loss: 0.4050 - val_acc: 0.9108\n",
      "Epoch 80/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3870 - acc: 0.9163 - val_loss: 0.4061 - val_acc: 0.9100\n",
      "Epoch 81/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3866 - acc: 0.9159 - val_loss: 0.4035 - val_acc: 0.9127\n",
      "Epoch 82/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3867 - acc: 0.9156 - val_loss: 0.4069 - val_acc: 0.9071\n",
      "Epoch 83/150\n",
      "33600/33600 [==============================] - 3s 93us/step - loss: 0.3865 - acc: 0.9154 - val_loss: 0.4073 - val_acc: 0.9100\n",
      "Epoch 84/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3860 - acc: 0.9161 - val_loss: 0.4065 - val_acc: 0.9088\n",
      "Epoch 85/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3861 - acc: 0.9159 - val_loss: 0.4049 - val_acc: 0.9094\n",
      "Epoch 86/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3864 - acc: 0.9151 - val_loss: 0.4050 - val_acc: 0.9092\n",
      "Epoch 87/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3863 - acc: 0.9159 - val_loss: 0.4059 - val_acc: 0.9098\n",
      "Epoch 88/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3864 - acc: 0.9152 - val_loss: 0.4062 - val_acc: 0.9102\n",
      "Epoch 89/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3867 - acc: 0.9156 - val_loss: 0.4065 - val_acc: 0.9086\n",
      "Epoch 90/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3865 - acc: 0.9150 - val_loss: 0.4048 - val_acc: 0.9094\n",
      "Epoch 91/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3865 - acc: 0.9151 - val_loss: 0.4068 - val_acc: 0.9104\n",
      "Epoch 92/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3867 - acc: 0.9159 - val_loss: 0.4039 - val_acc: 0.9110\n",
      "Epoch 93/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3868 - acc: 0.9158 - val_loss: 0.4042 - val_acc: 0.9104\n",
      "Epoch 94/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3862 - acc: 0.9162 - val_loss: 0.4076 - val_acc: 0.9080\n",
      "Epoch 95/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3868 - acc: 0.9161 - val_loss: 0.4054 - val_acc: 0.9107\n",
      "Epoch 96/150\n",
      "33600/33600 [==============================] - 3s 95us/step - loss: 0.3865 - acc: 0.9154 - val_loss: 0.4053 - val_acc: 0.9086\n",
      "Epoch 97/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3869 - acc: 0.9161 - val_loss: 0.4044 - val_acc: 0.9100\n",
      "Epoch 98/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3866 - acc: 0.9159 - val_loss: 0.4060 - val_acc: 0.9105\n",
      "Epoch 99/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.3871 - acc: 0.9161 - val_loss: 0.4046 - val_acc: 0.9098\n",
      "Epoch 100/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3873 - acc: 0.9153 - val_loss: 0.4029 - val_acc: 0.9113\n",
      "Epoch 101/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 0.3868 - acc: 0.9166 - val_loss: 0.4055 - val_acc: 0.9096\n",
      "Epoch 102/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3869 - acc: 0.9150 - val_loss: 0.4034 - val_acc: 0.9112\n",
      "Epoch 103/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3873 - acc: 0.9154 - val_loss: 0.4044 - val_acc: 0.9115\n",
      "Epoch 104/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3868 - acc: 0.9155 - val_loss: 0.4061 - val_acc: 0.9077\n",
      "Epoch 105/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3868 - acc: 0.9157 - val_loss: 0.4073 - val_acc: 0.9100\n",
      "Epoch 106/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 0.3866 - acc: 0.9162 - val_loss: 0.4080 - val_acc: 0.9099\n",
      "Epoch 107/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3870 - acc: 0.9163 - val_loss: 0.4043 - val_acc: 0.9098\n",
      "Epoch 108/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3866 - acc: 0.9163 - val_loss: 0.4071 - val_acc: 0.9089\n",
      "Epoch 109/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3865 - acc: 0.9160 - val_loss: 0.4082 - val_acc: 0.9100\n",
      "Epoch 110/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3870 - acc: 0.9154 - val_loss: 0.4044 - val_acc: 0.9095\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 111/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3869 - acc: 0.9161 - val_loss: 0.4075 - val_acc: 0.9065\n",
      "Epoch 112/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3869 - acc: 0.9160 - val_loss: 0.4049 - val_acc: 0.9107\n",
      "Epoch 113/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3865 - acc: 0.9167 - val_loss: 0.4045 - val_acc: 0.9119\n",
      "Epoch 114/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3868 - acc: 0.9164 - val_loss: 0.4080 - val_acc: 0.9065\n",
      "Epoch 115/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3869 - acc: 0.9153 - val_loss: 0.4049 - val_acc: 0.9101\n",
      "Epoch 116/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3870 - acc: 0.9160 - val_loss: 0.4074 - val_acc: 0.9088\n",
      "Epoch 117/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3869 - acc: 0.9162 - val_loss: 0.4047 - val_acc: 0.9095\n",
      "Epoch 118/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3866 - acc: 0.9158 - val_loss: 0.4037 - val_acc: 0.9119\n",
      "Epoch 119/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3866 - acc: 0.9159 - val_loss: 0.4052 - val_acc: 0.9108\n",
      "Epoch 120/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3867 - acc: 0.9160 - val_loss: 0.4053 - val_acc: 0.9102\n",
      "Epoch 121/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3867 - acc: 0.9164 - val_loss: 0.4051 - val_acc: 0.9095\n",
      "Epoch 122/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3866 - acc: 0.9157 - val_loss: 0.4095 - val_acc: 0.9046\n",
      "Epoch 123/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3867 - acc: 0.9157 - val_loss: 0.4051 - val_acc: 0.9094\n",
      "Epoch 124/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3872 - acc: 0.9156 - val_loss: 0.4063 - val_acc: 0.9104\n",
      "Epoch 125/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3869 - acc: 0.9160 - val_loss: 0.4047 - val_acc: 0.9101\n",
      "Epoch 126/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3863 - acc: 0.9167 - val_loss: 0.4064 - val_acc: 0.9069\n",
      "Epoch 127/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3870 - acc: 0.9151 - val_loss: 0.4040 - val_acc: 0.9108\n",
      "Epoch 128/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3866 - acc: 0.9155 - val_loss: 0.4079 - val_acc: 0.9049\n",
      "Epoch 129/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3869 - acc: 0.9165 - val_loss: 0.4055 - val_acc: 0.9096\n",
      "Epoch 130/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3871 - acc: 0.9168 - val_loss: 0.4059 - val_acc: 0.9087\n",
      "Epoch 131/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3872 - acc: 0.9154 - val_loss: 0.4050 - val_acc: 0.9108\n",
      "Epoch 132/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3870 - acc: 0.9158 - val_loss: 0.4075 - val_acc: 0.9088\n",
      "Epoch 133/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3871 - acc: 0.9161 - val_loss: 0.4048 - val_acc: 0.9098\n",
      "Epoch 134/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3867 - acc: 0.9164 - val_loss: 0.4074 - val_acc: 0.9060\n",
      "Epoch 135/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3866 - acc: 0.9162 - val_loss: 0.4044 - val_acc: 0.9089\n",
      "Epoch 136/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3864 - acc: 0.9152 - val_loss: 0.4055 - val_acc: 0.9094\n",
      "Epoch 137/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3870 - acc: 0.9151 - val_loss: 0.4049 - val_acc: 0.9101\n",
      "Epoch 138/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3868 - acc: 0.9167 - val_loss: 0.4081 - val_acc: 0.9062\n",
      "Epoch 139/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3864 - acc: 0.9157 - val_loss: 0.4101 - val_acc: 0.9062\n",
      "Epoch 140/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3866 - acc: 0.9160 - val_loss: 0.4086 - val_acc: 0.9082\n",
      "Epoch 141/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3863 - acc: 0.9159 - val_loss: 0.4076 - val_acc: 0.9062\n",
      "Epoch 142/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3868 - acc: 0.9153 - val_loss: 0.4068 - val_acc: 0.9099\n",
      "Epoch 143/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3867 - acc: 0.9157 - val_loss: 0.4038 - val_acc: 0.9114\n",
      "Epoch 144/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3868 - acc: 0.9155 - val_loss: 0.4071 - val_acc: 0.9087\n",
      "Epoch 145/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3869 - acc: 0.9155 - val_loss: 0.4046 - val_acc: 0.9092\n",
      "Epoch 146/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3868 - acc: 0.9156 - val_loss: 0.4047 - val_acc: 0.9110\n",
      "Epoch 147/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3867 - acc: 0.9148 - val_loss: 0.4053 - val_acc: 0.9094\n",
      "Epoch 148/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3871 - acc: 0.9163 - val_loss: 0.4066 - val_acc: 0.9083\n",
      "Epoch 149/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3871 - acc: 0.9156 - val_loss: 0.4042 - val_acc: 0.9085\n",
      "Epoch 150/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3867 - acc: 0.9157 - val_loss: 0.4059 - val_acc: 0.9094\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/150\n",
      "33600/33600 [==============================] - 4s 105us/step - loss: 0.6303 - acc: 0.8414 - val_loss: 0.3832 - val_acc: 0.8950\n",
      "Epoch 2/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3493 - acc: 0.9054 - val_loss: 0.3474 - val_acc: 0.9069\n",
      "Epoch 3/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3216 - acc: 0.9141 - val_loss: 0.3312 - val_acc: 0.9114\n",
      "Epoch 4/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3101 - acc: 0.9174 - val_loss: 0.3226 - val_acc: 0.9149\n",
      "Epoch 5/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3040 - acc: 0.9199 - val_loss: 0.3207 - val_acc: 0.9173\n",
      "Epoch 6/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2997 - acc: 0.9216 - val_loss: 0.3215 - val_acc: 0.9163\n",
      "Epoch 7/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2973 - acc: 0.9223 - val_loss: 0.3193 - val_acc: 0.9164\n",
      "Epoch 8/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2947 - acc: 0.9240 - val_loss: 0.3200 - val_acc: 0.9182\n",
      "Epoch 9/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2936 - acc: 0.9249 - val_loss: 0.3180 - val_acc: 0.9189\n",
      "Epoch 10/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2926 - acc: 0.9255 - val_loss: 0.3191 - val_acc: 0.9180\n",
      "Epoch 11/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2920 - acc: 0.9257 - val_loss: 0.3189 - val_acc: 0.9180\n",
      "Epoch 12/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2914 - acc: 0.9255 - val_loss: 0.3200 - val_acc: 0.9195\n",
      "Epoch 13/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2909 - acc: 0.9262 - val_loss: 0.3191 - val_acc: 0.9188\n",
      "Epoch 14/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2901 - acc: 0.9275 - val_loss: 0.3232 - val_acc: 0.9182\n",
      "Epoch 15/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2905 - acc: 0.9272 - val_loss: 0.3264 - val_acc: 0.9171\n",
      "Epoch 16/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2903 - acc: 0.9278 - val_loss: 0.3214 - val_acc: 0.9193\n",
      "Epoch 17/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2898 - acc: 0.9285 - val_loss: 0.3234 - val_acc: 0.9193\n",
      "Epoch 18/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2900 - acc: 0.9277 - val_loss: 0.3250 - val_acc: 0.9177\n",
      "Epoch 19/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2901 - acc: 0.9285 - val_loss: 0.3284 - val_acc: 0.9175\n",
      "Epoch 20/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2908 - acc: 0.9284 - val_loss: 0.3231 - val_acc: 0.9193\n",
      "Epoch 21/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2907 - acc: 0.9282 - val_loss: 0.3244 - val_acc: 0.9189\n",
      "Epoch 22/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2903 - acc: 0.9286 - val_loss: 0.3263 - val_acc: 0.9185\n",
      "Epoch 23/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2910 - acc: 0.9288 - val_loss: 0.3294 - val_acc: 0.9168\n",
      "Epoch 24/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2908 - acc: 0.9299 - val_loss: 0.3285 - val_acc: 0.9176\n",
      "Epoch 25/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2913 - acc: 0.9294 - val_loss: 0.3274 - val_acc: 0.9201\n",
      "Epoch 26/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2915 - acc: 0.9296 - val_loss: 0.3282 - val_acc: 0.9180\n",
      "Epoch 27/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2921 - acc: 0.9294 - val_loss: 0.3286 - val_acc: 0.9188\n",
      "Epoch 28/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2925 - acc: 0.9301 - val_loss: 0.3292 - val_acc: 0.9187\n",
      "Epoch 29/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2923 - acc: 0.9297 - val_loss: 0.3297 - val_acc: 0.9201\n",
      "Epoch 30/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2933 - acc: 0.9303 - val_loss: 0.3289 - val_acc: 0.9195\n",
      "Epoch 31/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2935 - acc: 0.9297 - val_loss: 0.3302 - val_acc: 0.9207\n",
      "Epoch 32/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2941 - acc: 0.9296 - val_loss: 0.3320 - val_acc: 0.9196\n",
      "Epoch 33/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.2944 - acc: 0.9293 - val_loss: 0.3323 - val_acc: 0.9195\n",
      "Epoch 34/150\n",
      "33600/33600 [==============================] - 3s 92us/step - loss: 0.2947 - acc: 0.9307 - val_loss: 0.3341 - val_acc: 0.9188\n",
      "Epoch 35/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.2953 - acc: 0.9302 - val_loss: 0.3318 - val_acc: 0.9193\n",
      "Epoch 36/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2959 - acc: 0.9301 - val_loss: 0.3344 - val_acc: 0.9186\n",
      "Epoch 37/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2964 - acc: 0.9303 - val_loss: 0.3351 - val_acc: 0.9193\n",
      "Epoch 38/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2967 - acc: 0.9302 - val_loss: 0.3357 - val_acc: 0.9200\n",
      "Epoch 39/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2970 - acc: 0.9307 - val_loss: 0.3372 - val_acc: 0.9188\n",
      "Epoch 40/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2973 - acc: 0.9303 - val_loss: 0.3375 - val_acc: 0.9195\n",
      "Epoch 41/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2974 - acc: 0.9302 - val_loss: 0.3384 - val_acc: 0.9183\n",
      "Epoch 42/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.2981 - acc: 0.9304 - val_loss: 0.3390 - val_acc: 0.9174\n",
      "Epoch 43/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2986 - acc: 0.9305 - val_loss: 0.3380 - val_acc: 0.9208\n",
      "Epoch 44/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2992 - acc: 0.9296 - val_loss: 0.3379 - val_acc: 0.9202\n",
      "Epoch 45/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2996 - acc: 0.9297 - val_loss: 0.3411 - val_acc: 0.9181\n",
      "Epoch 46/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2997 - acc: 0.9301 - val_loss: 0.3415 - val_acc: 0.9179\n",
      "Epoch 47/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3004 - acc: 0.9304 - val_loss: 0.3416 - val_acc: 0.9193\n",
      "Epoch 48/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3011 - acc: 0.9292 - val_loss: 0.3417 - val_acc: 0.9185\n",
      "Epoch 49/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3013 - acc: 0.9305 - val_loss: 0.3424 - val_acc: 0.9198\n",
      "Epoch 50/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3015 - acc: 0.9304 - val_loss: 0.3434 - val_acc: 0.9185\n",
      "Epoch 51/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3018 - acc: 0.9312 - val_loss: 0.3425 - val_acc: 0.9187\n",
      "Epoch 52/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3026 - acc: 0.9300 - val_loss: 0.3458 - val_acc: 0.9186\n",
      "Epoch 53/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3028 - acc: 0.9296 - val_loss: 0.3435 - val_acc: 0.9177\n",
      "Epoch 54/150\n",
      "33600/33600 [==============================] - 3s 93us/step - loss: 0.3031 - acc: 0.9299 - val_loss: 0.3438 - val_acc: 0.9204\n",
      "Epoch 55/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3034 - acc: 0.9308 - val_loss: 0.3464 - val_acc: 0.9181\n",
      "Epoch 56/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3035 - acc: 0.9306 - val_loss: 0.3475 - val_acc: 0.9177\n",
      "Epoch 57/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3047 - acc: 0.9304 - val_loss: 0.3473 - val_acc: 0.9190\n",
      "Epoch 58/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3043 - acc: 0.9312 - val_loss: 0.3469 - val_acc: 0.9175\n",
      "Epoch 59/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3051 - acc: 0.9308 - val_loss: 0.3482 - val_acc: 0.9185\n",
      "Epoch 60/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3056 - acc: 0.9306 - val_loss: 0.3481 - val_acc: 0.9180\n",
      "Epoch 61/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3057 - acc: 0.9304 - val_loss: 0.3491 - val_acc: 0.9175\n",
      "Epoch 62/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3061 - acc: 0.9304 - val_loss: 0.3488 - val_acc: 0.9186\n",
      "Epoch 63/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3068 - acc: 0.9301 - val_loss: 0.3491 - val_acc: 0.9192\n",
      "Epoch 64/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3071 - acc: 0.9302 - val_loss: 0.3503 - val_acc: 0.9187\n",
      "Epoch 65/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3081 - acc: 0.9302 - val_loss: 0.3513 - val_acc: 0.9183\n",
      "Epoch 66/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3075 - acc: 0.9300 - val_loss: 0.3509 - val_acc: 0.9189\n",
      "Epoch 67/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3082 - acc: 0.9310 - val_loss: 0.3507 - val_acc: 0.9181\n",
      "Epoch 68/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3082 - acc: 0.9312 - val_loss: 0.3514 - val_acc: 0.9175\n",
      "Epoch 69/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3089 - acc: 0.9307 - val_loss: 0.3508 - val_acc: 0.9176\n",
      "Epoch 70/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3094 - acc: 0.9304 - val_loss: 0.3526 - val_acc: 0.9180\n",
      "Epoch 71/150\n",
      "33600/33600 [==============================] - 3s 96us/step - loss: 0.3098 - acc: 0.9304 - val_loss: 0.3525 - val_acc: 0.9185\n",
      "Epoch 72/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3103 - acc: 0.9303 - val_loss: 0.3538 - val_acc: 0.9179\n",
      "Epoch 73/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3105 - acc: 0.9307 - val_loss: 0.3521 - val_acc: 0.9181\n",
      "Epoch 74/150\n",
      "33600/33600 [==============================] - 3s 95us/step - loss: 0.3112 - acc: 0.9310 - val_loss: 0.3561 - val_acc: 0.9168\n",
      "Epoch 75/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3107 - acc: 0.9301 - val_loss: 0.3555 - val_acc: 0.9167\n",
      "Epoch 76/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3114 - acc: 0.9303 - val_loss: 0.3535 - val_acc: 0.9202\n",
      "Epoch 77/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3118 - acc: 0.9301 - val_loss: 0.3576 - val_acc: 0.9161\n",
      "Epoch 78/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3126 - acc: 0.9303 - val_loss: 0.3558 - val_acc: 0.9175\n",
      "Epoch 79/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3128 - acc: 0.9300 - val_loss: 0.3572 - val_acc: 0.9190\n",
      "Epoch 80/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3128 - acc: 0.9308 - val_loss: 0.3567 - val_acc: 0.9180\n",
      "Epoch 81/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3130 - acc: 0.9305 - val_loss: 0.3562 - val_acc: 0.9185\n",
      "Epoch 82/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3134 - acc: 0.9306 - val_loss: 0.3568 - val_acc: 0.9175\n",
      "Epoch 83/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3140 - acc: 0.9305 - val_loss: 0.3573 - val_acc: 0.9194\n",
      "Epoch 84/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3137 - acc: 0.9310 - val_loss: 0.3602 - val_acc: 0.9173\n",
      "Epoch 85/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3142 - acc: 0.9315 - val_loss: 0.3556 - val_acc: 0.9195\n",
      "Epoch 86/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3146 - acc: 0.9308 - val_loss: 0.3597 - val_acc: 0.9189\n",
      "Epoch 87/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3149 - acc: 0.9307 - val_loss: 0.3595 - val_acc: 0.9183\n",
      "Epoch 88/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3151 - acc: 0.9309 - val_loss: 0.3609 - val_acc: 0.9157\n",
      "Epoch 89/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3155 - acc: 0.9307 - val_loss: 0.3614 - val_acc: 0.9186\n",
      "Epoch 90/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3156 - acc: 0.9310 - val_loss: 0.3606 - val_acc: 0.9173\n",
      "Epoch 91/150\n",
      "33600/33600 [==============================] - 3s 82us/step - loss: 0.3158 - acc: 0.9307 - val_loss: 0.3607 - val_acc: 0.9193\n",
      "Epoch 92/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3165 - acc: 0.9308 - val_loss: 0.3611 - val_acc: 0.9176\n",
      "Epoch 93/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3168 - acc: 0.9301 - val_loss: 0.3605 - val_acc: 0.9187\n",
      "Epoch 94/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3166 - acc: 0.9306 - val_loss: 0.3611 - val_acc: 0.9181\n",
      "Epoch 95/150\n",
      "33600/33600 [==============================] - 3s 82us/step - loss: 0.3174 - acc: 0.9313 - val_loss: 0.3614 - val_acc: 0.9181\n",
      "Epoch 96/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3176 - acc: 0.9310 - val_loss: 0.3610 - val_acc: 0.9188\n",
      "Epoch 97/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3183 - acc: 0.9310 - val_loss: 0.3652 - val_acc: 0.9155\n",
      "Epoch 98/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3184 - acc: 0.9304 - val_loss: 0.3610 - val_acc: 0.9199\n",
      "Epoch 99/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3192 - acc: 0.9310 - val_loss: 0.3626 - val_acc: 0.9181\n",
      "Epoch 100/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3198 - acc: 0.9300 - val_loss: 0.3620 - val_acc: 0.9182\n",
      "Epoch 101/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3197 - acc: 0.9307 - val_loss: 0.3631 - val_acc: 0.9176\n",
      "Epoch 102/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3198 - acc: 0.9305 - val_loss: 0.3655 - val_acc: 0.9164\n",
      "Epoch 103/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3203 - acc: 0.9301 - val_loss: 0.3629 - val_acc: 0.9182\n",
      "Epoch 104/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3204 - acc: 0.9303 - val_loss: 0.3635 - val_acc: 0.9189\n",
      "Epoch 105/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3210 - acc: 0.9304 - val_loss: 0.3636 - val_acc: 0.9177\n",
      "Epoch 106/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3216 - acc: 0.9310 - val_loss: 0.3653 - val_acc: 0.9180\n",
      "Epoch 107/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3216 - acc: 0.9307 - val_loss: 0.3646 - val_acc: 0.9180\n",
      "Epoch 108/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3215 - acc: 0.9302 - val_loss: 0.3662 - val_acc: 0.9180\n",
      "Epoch 109/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3221 - acc: 0.9306 - val_loss: 0.3651 - val_acc: 0.9180\n",
      "Epoch 110/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3222 - acc: 0.9299 - val_loss: 0.3651 - val_acc: 0.9193\n",
      "Epoch 111/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3222 - acc: 0.9301 - val_loss: 0.3674 - val_acc: 0.9175\n",
      "Epoch 112/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3226 - acc: 0.9309 - val_loss: 0.3687 - val_acc: 0.9158\n",
      "Epoch 113/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3230 - acc: 0.9306 - val_loss: 0.3653 - val_acc: 0.9179\n",
      "Epoch 114/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3231 - acc: 0.9307 - val_loss: 0.3659 - val_acc: 0.9192\n",
      "Epoch 115/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3233 - acc: 0.9310 - val_loss: 0.3685 - val_acc: 0.9176\n",
      "Epoch 116/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3237 - acc: 0.9306 - val_loss: 0.3683 - val_acc: 0.9185\n",
      "Epoch 117/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3239 - acc: 0.9309 - val_loss: 0.3678 - val_acc: 0.9181\n",
      "Epoch 118/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3234 - acc: 0.9307 - val_loss: 0.3685 - val_acc: 0.9155\n",
      "Epoch 119/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3240 - acc: 0.9309 - val_loss: 0.3687 - val_acc: 0.9174\n",
      "Epoch 120/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3236 - acc: 0.9305 - val_loss: 0.3666 - val_acc: 0.9185\n",
      "Epoch 121/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3242 - acc: 0.9307 - val_loss: 0.3668 - val_acc: 0.9195\n",
      "Epoch 122/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3240 - acc: 0.9303 - val_loss: 0.3696 - val_acc: 0.9177\n",
      "Epoch 123/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3242 - acc: 0.9307 - val_loss: 0.3682 - val_acc: 0.9183\n",
      "Epoch 124/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3243 - acc: 0.9308 - val_loss: 0.3690 - val_acc: 0.9176\n",
      "Epoch 125/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3247 - acc: 0.9305 - val_loss: 0.3694 - val_acc: 0.9169\n",
      "Epoch 126/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3247 - acc: 0.9306 - val_loss: 0.3682 - val_acc: 0.9173\n",
      "Epoch 127/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3241 - acc: 0.9313 - val_loss: 0.3679 - val_acc: 0.9181\n",
      "Epoch 128/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3250 - acc: 0.9307 - val_loss: 0.3707 - val_acc: 0.9167\n",
      "Epoch 129/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3250 - acc: 0.9312 - val_loss: 0.3688 - val_acc: 0.9185\n",
      "Epoch 130/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3254 - acc: 0.9305 - val_loss: 0.3693 - val_acc: 0.9185\n",
      "Epoch 131/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3251 - acc: 0.9310 - val_loss: 0.3711 - val_acc: 0.9179\n",
      "Epoch 132/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3249 - acc: 0.9316 - val_loss: 0.3720 - val_acc: 0.9155\n",
      "Epoch 133/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.3253 - acc: 0.9306 - val_loss: 0.3705 - val_acc: 0.9175\n",
      "Epoch 134/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3256 - acc: 0.9317 - val_loss: 0.3712 - val_acc: 0.9180\n",
      "Epoch 135/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.3258 - acc: 0.9308 - val_loss: 0.3717 - val_acc: 0.9164\n",
      "Epoch 136/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.3256 - acc: 0.9305 - val_loss: 0.3707 - val_acc: 0.9171\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 137/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3262 - acc: 0.9307 - val_loss: 0.3689 - val_acc: 0.9186\n",
      "Epoch 138/150\n",
      "33600/33600 [==============================] - 3s 90us/step - loss: 0.3258 - acc: 0.9304 - val_loss: 0.3692 - val_acc: 0.9194\n",
      "Epoch 139/150\n",
      "33600/33600 [==============================] - 3s 94us/step - loss: 0.3265 - acc: 0.9310 - val_loss: 0.3690 - val_acc: 0.9182\n",
      "Epoch 140/150\n",
      "33600/33600 [==============================] - 3s 99us/step - loss: 0.3266 - acc: 0.9309 - val_loss: 0.3719 - val_acc: 0.9174\n",
      "Epoch 141/150\n",
      "33600/33600 [==============================] - 3s 99us/step - loss: 0.3269 - acc: 0.9307 - val_loss: 0.3718 - val_acc: 0.9175\n",
      "Epoch 142/150\n",
      "33600/33600 [==============================] - 3s 91us/step - loss: 0.3263 - acc: 0.9318 - val_loss: 0.3712 - val_acc: 0.9175\n",
      "Epoch 143/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3272 - acc: 0.9309 - val_loss: 0.3708 - val_acc: 0.9187\n",
      "Epoch 144/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3273 - acc: 0.9308 - val_loss: 0.3712 - val_acc: 0.9180\n",
      "Epoch 145/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3275 - acc: 0.9306 - val_loss: 0.3716 - val_acc: 0.9186\n",
      "Epoch 146/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3274 - acc: 0.9312 - val_loss: 0.3711 - val_acc: 0.9188\n",
      "Epoch 147/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.3281 - acc: 0.9313 - val_loss: 0.3717 - val_acc: 0.9188\n",
      "Epoch 148/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3285 - acc: 0.9306 - val_loss: 0.3722 - val_acc: 0.9194\n",
      "Epoch 149/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.3283 - acc: 0.9306 - val_loss: 0.3735 - val_acc: 0.9195\n",
      "Epoch 150/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3284 - acc: 0.9309 - val_loss: 0.3733 - val_acc: 0.9195\n",
      "Train on 33600 samples, validate on 8400 samples\n",
      "Epoch 1/150\n",
      "33600/33600 [==============================] - 4s 106us/step - loss: 0.6256 - acc: 0.8427 - val_loss: 0.3724 - val_acc: 0.8955\n",
      "Epoch 2/150\n",
      "33600/33600 [==============================] - 3s 89us/step - loss: 0.3356 - acc: 0.9063 - val_loss: 0.3251 - val_acc: 0.9068\n",
      "Epoch 3/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.3024 - acc: 0.9150 - val_loss: 0.3076 - val_acc: 0.9139\n",
      "Epoch 4/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2871 - acc: 0.9198 - val_loss: 0.2993 - val_acc: 0.9149\n",
      "Epoch 5/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2772 - acc: 0.9226 - val_loss: 0.2955 - val_acc: 0.9185\n",
      "Epoch 6/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2713 - acc: 0.9251 - val_loss: 0.2896 - val_acc: 0.9201\n",
      "Epoch 7/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2668 - acc: 0.9263 - val_loss: 0.2891 - val_acc: 0.9200\n",
      "Epoch 8/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2627 - acc: 0.9267 - val_loss: 0.2889 - val_acc: 0.9188\n",
      "Epoch 9/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2595 - acc: 0.9285 - val_loss: 0.2861 - val_acc: 0.9205\n",
      "Epoch 10/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2570 - acc: 0.9299 - val_loss: 0.2843 - val_acc: 0.9225\n",
      "Epoch 11/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2546 - acc: 0.9299 - val_loss: 0.2852 - val_acc: 0.9214\n",
      "Epoch 12/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2535 - acc: 0.9304 - val_loss: 0.2842 - val_acc: 0.9225\n",
      "Epoch 13/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2516 - acc: 0.9321 - val_loss: 0.2855 - val_acc: 0.9226\n",
      "Epoch 14/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2496 - acc: 0.9317 - val_loss: 0.2827 - val_acc: 0.9230\n",
      "Epoch 15/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2493 - acc: 0.9332 - val_loss: 0.2843 - val_acc: 0.9221\n",
      "Epoch 16/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2477 - acc: 0.9335 - val_loss: 0.2842 - val_acc: 0.9240\n",
      "Epoch 17/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2471 - acc: 0.9339 - val_loss: 0.2851 - val_acc: 0.9233\n",
      "Epoch 18/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2458 - acc: 0.9343 - val_loss: 0.2892 - val_acc: 0.9215\n",
      "Epoch 19/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2451 - acc: 0.9333 - val_loss: 0.2856 - val_acc: 0.9238\n",
      "Epoch 20/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2439 - acc: 0.9343 - val_loss: 0.2845 - val_acc: 0.9242\n",
      "Epoch 21/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2429 - acc: 0.9345 - val_loss: 0.2852 - val_acc: 0.9226\n",
      "Epoch 22/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2428 - acc: 0.9345 - val_loss: 0.2851 - val_acc: 0.9242\n",
      "Epoch 23/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2423 - acc: 0.9356 - val_loss: 0.2871 - val_acc: 0.9211\n",
      "Epoch 24/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2416 - acc: 0.9348 - val_loss: 0.2866 - val_acc: 0.9248\n",
      "Epoch 25/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2412 - acc: 0.9357 - val_loss: 0.2880 - val_acc: 0.9240\n",
      "Epoch 26/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2405 - acc: 0.9360 - val_loss: 0.2886 - val_acc: 0.9245\n",
      "Epoch 27/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2398 - acc: 0.9363 - val_loss: 0.2870 - val_acc: 0.9242\n",
      "Epoch 28/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2394 - acc: 0.9360 - val_loss: 0.2887 - val_acc: 0.9236\n",
      "Epoch 29/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2390 - acc: 0.9360 - val_loss: 0.2887 - val_acc: 0.9244\n",
      "Epoch 30/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2386 - acc: 0.9367 - val_loss: 0.2897 - val_acc: 0.9243\n",
      "Epoch 31/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2384 - acc: 0.9371 - val_loss: 0.2894 - val_acc: 0.9230\n",
      "Epoch 32/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2377 - acc: 0.9366 - val_loss: 0.2901 - val_acc: 0.9246\n",
      "Epoch 33/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2376 - acc: 0.9371 - val_loss: 0.2904 - val_acc: 0.9252\n",
      "Epoch 34/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2372 - acc: 0.9374 - val_loss: 0.2926 - val_acc: 0.9231\n",
      "Epoch 35/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2370 - acc: 0.9372 - val_loss: 0.2906 - val_acc: 0.9245\n",
      "Epoch 36/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2367 - acc: 0.9373 - val_loss: 0.2916 - val_acc: 0.9262\n",
      "Epoch 37/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2367 - acc: 0.9377 - val_loss: 0.2929 - val_acc: 0.9237\n",
      "Epoch 38/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2366 - acc: 0.9378 - val_loss: 0.2924 - val_acc: 0.9238\n",
      "Epoch 39/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2359 - acc: 0.9384 - val_loss: 0.2944 - val_acc: 0.9246\n",
      "Epoch 40/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2357 - acc: 0.9381 - val_loss: 0.2950 - val_acc: 0.9244\n",
      "Epoch 41/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2355 - acc: 0.9386 - val_loss: 0.2939 - val_acc: 0.9245\n",
      "Epoch 42/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2354 - acc: 0.9383 - val_loss: 0.2959 - val_acc: 0.9224\n",
      "Epoch 43/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2348 - acc: 0.9385 - val_loss: 0.2969 - val_acc: 0.9240\n",
      "Epoch 44/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2347 - acc: 0.9389 - val_loss: 0.3001 - val_acc: 0.9201\n",
      "Epoch 45/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2341 - acc: 0.9385 - val_loss: 0.2969 - val_acc: 0.9239\n",
      "Epoch 46/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2345 - acc: 0.9388 - val_loss: 0.2969 - val_acc: 0.9231\n",
      "Epoch 47/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2340 - acc: 0.9386 - val_loss: 0.2993 - val_acc: 0.9238\n",
      "Epoch 48/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2338 - acc: 0.9387 - val_loss: 0.3004 - val_acc: 0.9225\n",
      "Epoch 49/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2338 - acc: 0.9392 - val_loss: 0.3021 - val_acc: 0.9224\n",
      "Epoch 50/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2336 - acc: 0.9399 - val_loss: 0.3002 - val_acc: 0.9227\n",
      "Epoch 51/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2334 - acc: 0.9396 - val_loss: 0.2999 - val_acc: 0.9239\n",
      "Epoch 52/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2337 - acc: 0.9397 - val_loss: 0.3026 - val_acc: 0.9230\n",
      "Epoch 53/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2332 - acc: 0.9400 - val_loss: 0.3008 - val_acc: 0.9237\n",
      "Epoch 54/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2330 - acc: 0.9393 - val_loss: 0.3042 - val_acc: 0.9225\n",
      "Epoch 55/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2325 - acc: 0.9396 - val_loss: 0.3032 - val_acc: 0.9243\n",
      "Epoch 56/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2331 - acc: 0.9400 - val_loss: 0.3012 - val_acc: 0.9235\n",
      "Epoch 57/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2321 - acc: 0.9406 - val_loss: 0.3023 - val_acc: 0.9227\n",
      "Epoch 58/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2324 - acc: 0.9408 - val_loss: 0.3032 - val_acc: 0.9232\n",
      "Epoch 59/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2320 - acc: 0.9399 - val_loss: 0.3047 - val_acc: 0.9221\n",
      "Epoch 60/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2321 - acc: 0.9401 - val_loss: 0.3037 - val_acc: 0.9240\n",
      "Epoch 61/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2315 - acc: 0.9413 - val_loss: 0.3052 - val_acc: 0.9231\n",
      "Epoch 62/150\n",
      "33600/33600 [==============================] - 3s 83us/step - loss: 0.2314 - acc: 0.9403 - val_loss: 0.3055 - val_acc: 0.9232\n",
      "Epoch 63/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2313 - acc: 0.9400 - val_loss: 0.3094 - val_acc: 0.9229\n",
      "Epoch 64/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2312 - acc: 0.9410 - val_loss: 0.3061 - val_acc: 0.9240\n",
      "Epoch 65/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2311 - acc: 0.9401 - val_loss: 0.3091 - val_acc: 0.9208\n",
      "Epoch 66/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2308 - acc: 0.9403 - val_loss: 0.3082 - val_acc: 0.9237\n",
      "Epoch 67/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2308 - acc: 0.9409 - val_loss: 0.3083 - val_acc: 0.9232\n",
      "Epoch 68/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2304 - acc: 0.9413 - val_loss: 0.3089 - val_acc: 0.9231\n",
      "Epoch 69/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2305 - acc: 0.9404 - val_loss: 0.3091 - val_acc: 0.9233\n",
      "Epoch 70/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2301 - acc: 0.9409 - val_loss: 0.3106 - val_acc: 0.9229\n",
      "Epoch 71/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2300 - acc: 0.9413 - val_loss: 0.3107 - val_acc: 0.9230\n",
      "Epoch 72/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2301 - acc: 0.9412 - val_loss: 0.3105 - val_acc: 0.9233\n",
      "Epoch 73/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2302 - acc: 0.9404 - val_loss: 0.3115 - val_acc: 0.9219\n",
      "Epoch 74/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2297 - acc: 0.9412 - val_loss: 0.3125 - val_acc: 0.9219\n",
      "Epoch 75/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2296 - acc: 0.9414 - val_loss: 0.3119 - val_acc: 0.9208\n",
      "Epoch 76/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2293 - acc: 0.9417 - val_loss: 0.3139 - val_acc: 0.9200\n",
      "Epoch 77/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2294 - acc: 0.9417 - val_loss: 0.3145 - val_acc: 0.9230\n",
      "Epoch 78/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2299 - acc: 0.9415 - val_loss: 0.3167 - val_acc: 0.9211\n",
      "Epoch 79/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2293 - acc: 0.9418 - val_loss: 0.3138 - val_acc: 0.9238\n",
      "Epoch 80/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2294 - acc: 0.9416 - val_loss: 0.3146 - val_acc: 0.9218\n",
      "Epoch 81/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2292 - acc: 0.9421 - val_loss: 0.3160 - val_acc: 0.9232\n",
      "Epoch 82/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2294 - acc: 0.9413 - val_loss: 0.3184 - val_acc: 0.9204\n",
      "Epoch 83/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2283 - acc: 0.9424 - val_loss: 0.3164 - val_acc: 0.9210\n",
      "Epoch 84/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2288 - acc: 0.9414 - val_loss: 0.3166 - val_acc: 0.9230\n",
      "Epoch 85/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2287 - acc: 0.9422 - val_loss: 0.3198 - val_acc: 0.9192\n",
      "Epoch 86/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2289 - acc: 0.9418 - val_loss: 0.3192 - val_acc: 0.9225\n",
      "Epoch 87/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2289 - acc: 0.9416 - val_loss: 0.3183 - val_acc: 0.9213\n",
      "Epoch 88/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2289 - acc: 0.9422 - val_loss: 0.3182 - val_acc: 0.9219\n",
      "Epoch 89/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2283 - acc: 0.9419 - val_loss: 0.3202 - val_acc: 0.9207\n",
      "Epoch 90/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2285 - acc: 0.9418 - val_loss: 0.3183 - val_acc: 0.9223\n",
      "Epoch 91/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2284 - acc: 0.9423 - val_loss: 0.3206 - val_acc: 0.9219\n",
      "Epoch 92/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2280 - acc: 0.9424 - val_loss: 0.3214 - val_acc: 0.9204\n",
      "Epoch 93/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2282 - acc: 0.9421 - val_loss: 0.3210 - val_acc: 0.9213\n",
      "Epoch 94/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2280 - acc: 0.9420 - val_loss: 0.3196 - val_acc: 0.9211\n",
      "Epoch 95/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2280 - acc: 0.9431 - val_loss: 0.3226 - val_acc: 0.9218\n",
      "Epoch 96/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2278 - acc: 0.9423 - val_loss: 0.3213 - val_acc: 0.9192\n",
      "Epoch 97/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2278 - acc: 0.9426 - val_loss: 0.3218 - val_acc: 0.9224\n",
      "Epoch 98/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2279 - acc: 0.9425 - val_loss: 0.3220 - val_acc: 0.9214\n",
      "Epoch 99/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2277 - acc: 0.9426 - val_loss: 0.3228 - val_acc: 0.9223\n",
      "Epoch 100/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2275 - acc: 0.9428 - val_loss: 0.3237 - val_acc: 0.9198\n",
      "Epoch 101/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2275 - acc: 0.9434 - val_loss: 0.3242 - val_acc: 0.9207\n",
      "Epoch 102/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2277 - acc: 0.9428 - val_loss: 0.3241 - val_acc: 0.9214\n",
      "Epoch 103/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2272 - acc: 0.9428 - val_loss: 0.3267 - val_acc: 0.9208\n",
      "Epoch 104/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2276 - acc: 0.9424 - val_loss: 0.3251 - val_acc: 0.9198\n",
      "Epoch 105/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2275 - acc: 0.9429 - val_loss: 0.3244 - val_acc: 0.9199\n",
      "Epoch 106/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2270 - acc: 0.9424 - val_loss: 0.3254 - val_acc: 0.9215\n",
      "Epoch 107/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2270 - acc: 0.9422 - val_loss: 0.3254 - val_acc: 0.9199\n",
      "Epoch 108/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2270 - acc: 0.9436 - val_loss: 0.3269 - val_acc: 0.9201\n",
      "Epoch 109/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2272 - acc: 0.9428 - val_loss: 0.3254 - val_acc: 0.9199\n",
      "Epoch 110/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2267 - acc: 0.9432 - val_loss: 0.3272 - val_acc: 0.9212\n",
      "Epoch 111/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2269 - acc: 0.9433 - val_loss: 0.3274 - val_acc: 0.9219\n",
      "Epoch 112/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2269 - acc: 0.9437 - val_loss: 0.3275 - val_acc: 0.9202\n",
      "Epoch 113/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2267 - acc: 0.9434 - val_loss: 0.3303 - val_acc: 0.9199\n",
      "Epoch 114/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2270 - acc: 0.9434 - val_loss: 0.3286 - val_acc: 0.9218\n",
      "Epoch 115/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2267 - acc: 0.9432 - val_loss: 0.3305 - val_acc: 0.9206\n",
      "Epoch 116/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2268 - acc: 0.9429 - val_loss: 0.3293 - val_acc: 0.9204\n",
      "Epoch 117/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2267 - acc: 0.9431 - val_loss: 0.3292 - val_acc: 0.9217\n",
      "Epoch 118/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2268 - acc: 0.9438 - val_loss: 0.3309 - val_acc: 0.9190\n",
      "Epoch 119/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2268 - acc: 0.9432 - val_loss: 0.3286 - val_acc: 0.9224\n",
      "Epoch 120/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2266 - acc: 0.9434 - val_loss: 0.3331 - val_acc: 0.9198\n",
      "Epoch 121/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2270 - acc: 0.9434 - val_loss: 0.3336 - val_acc: 0.9195\n",
      "Epoch 122/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2263 - acc: 0.9436 - val_loss: 0.3377 - val_acc: 0.9188\n",
      "Epoch 123/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2265 - acc: 0.9434 - val_loss: 0.3328 - val_acc: 0.9204\n",
      "Epoch 124/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2263 - acc: 0.9441 - val_loss: 0.3409 - val_acc: 0.9174\n",
      "Epoch 125/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2266 - acc: 0.9431 - val_loss: 0.3337 - val_acc: 0.9199\n",
      "Epoch 126/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2265 - acc: 0.9429 - val_loss: 0.3338 - val_acc: 0.9204\n",
      "Epoch 127/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2265 - acc: 0.9435 - val_loss: 0.3337 - val_acc: 0.9210\n",
      "Epoch 128/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2263 - acc: 0.9439 - val_loss: 0.3391 - val_acc: 0.9185\n",
      "Epoch 129/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2261 - acc: 0.9432 - val_loss: 0.3334 - val_acc: 0.9208\n",
      "Epoch 130/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2262 - acc: 0.9449 - val_loss: 0.3358 - val_acc: 0.9215\n",
      "Epoch 131/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2262 - acc: 0.9444 - val_loss: 0.3399 - val_acc: 0.9173\n",
      "Epoch 132/150\n",
      "33600/33600 [==============================] - 3s 87us/step - loss: 0.2260 - acc: 0.9439 - val_loss: 0.3384 - val_acc: 0.9193\n",
      "Epoch 133/150\n",
      "33600/33600 [==============================] - 3s 88us/step - loss: 0.2258 - acc: 0.9440 - val_loss: 0.3361 - val_acc: 0.9200\n",
      "Epoch 134/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2261 - acc: 0.9438 - val_loss: 0.3396 - val_acc: 0.9171\n",
      "Epoch 135/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2261 - acc: 0.9437 - val_loss: 0.3367 - val_acc: 0.9202\n",
      "Epoch 136/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2257 - acc: 0.9444 - val_loss: 0.3400 - val_acc: 0.9180\n",
      "Epoch 137/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2259 - acc: 0.9444 - val_loss: 0.3390 - val_acc: 0.9201\n",
      "Epoch 138/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2259 - acc: 0.9431 - val_loss: 0.3359 - val_acc: 0.9210\n",
      "Epoch 139/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2257 - acc: 0.9443 - val_loss: 0.3376 - val_acc: 0.9208\n",
      "Epoch 140/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2254 - acc: 0.9441 - val_loss: 0.3377 - val_acc: 0.9202\n",
      "Epoch 141/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2255 - acc: 0.9444 - val_loss: 0.3370 - val_acc: 0.9208\n",
      "Epoch 142/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2256 - acc: 0.9435 - val_loss: 0.3401 - val_acc: 0.9199\n",
      "Epoch 143/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2256 - acc: 0.9443 - val_loss: 0.3384 - val_acc: 0.9194\n",
      "Epoch 144/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2253 - acc: 0.9437 - val_loss: 0.3385 - val_acc: 0.9215\n",
      "Epoch 145/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2254 - acc: 0.9438 - val_loss: 0.3385 - val_acc: 0.9206\n",
      "Epoch 146/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2258 - acc: 0.9440 - val_loss: 0.3385 - val_acc: 0.9204\n",
      "Epoch 147/150\n",
      "33600/33600 [==============================] - 3s 86us/step - loss: 0.2258 - acc: 0.9446 - val_loss: 0.3402 - val_acc: 0.9202\n",
      "Epoch 148/150\n",
      "33600/33600 [==============================] - 3s 85us/step - loss: 0.2252 - acc: 0.9440 - val_loss: 0.3397 - val_acc: 0.9194\n",
      "Epoch 149/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2251 - acc: 0.9446 - val_loss: 0.3404 - val_acc: 0.9196\n",
      "Epoch 150/150\n",
      "33600/33600 [==============================] - 3s 84us/step - loss: 0.2254 - acc: 0.9448 - val_loss: 0.3431 - val_acc: 0.9195\n"
     ]
    }
   ],
   "source": [
    "regs = [0.1,0.01,0.001,0.0001,0]\n",
    "models = []\n",
    "optimizer = 'rmsprop'\n",
    "histories = []\n",
    "for reg in regs:\n",
    "    model,history = linear_model(optimizer,150, train_x,train_y,regularization=reg,valid_x=valid_x,valid_y=valid_y)\n",
    "    models.append(model)\n",
    "    histories.append(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "regularizaiton training acc validation acc\n",
      "0.1 0.837738095238 0.837023809524\n",
      "0.01 0.889285714286 0.8875\n",
      "0.001 0.915654761905 0.909404761905\n",
      "0.0001 0.930863095238 0.919523809524\n",
      "0 0.944761904762 0.919523809524\n"
     ]
    }
   ],
   "source": [
    "print('regularizaiton','training acc','validation acc')\n",
    "for idx, histor in enumerate(histories):\n",
    "    tr_acc    = histor.history['acc'][-1]\n",
    "    valid_acc = histor.history['val_acc'][-1]\n",
    "    print(regs[idx],tr_acc,valid_acc)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "## the best model is the one with 0.0001 reg\n",
    "model = models[-1]\n",
    "generate_submission(model,test_data,'sub_rmsprop_reg0_150ep_94.4tr_91.9valid.csv',['ImageId','Label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Regularization  | Training acc  | Validation acc  |\n",
    "| -------------   |:-------------:| ---------------:|\n",
    "| 0.1             | 83.9          |    83.24        |\n",
    "| 0.01            | 8899          |    8852         |\n",
    "| 0.001           | 9159          |    9114         |"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}