4- Lab6 MNIST Beginner to Advanced - Image Augmentation and Ensemble of Convolutional Neural Networks Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building an ensemble of classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.3\n",
    "set_session(tf.Session(config=config))\n",
    "\n",
    "notebook_num = 4\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "path = 'models/'+str(notebook_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def history_appending(history1,history2):\n",
    "    history1.history['acc'] += history2.history['acc']\n",
    "    history1.history['val_acc'] += history2.history['val_acc']\n",
    "    history1.history['val_loss'] += history2.history['val_loss']\n",
    "    history1.history['loss'] += history2.history['loss']\n",
    "    return history1\n",
    "\n",
    "def history_from_list(history_list):\n",
    "    for i in range(len(history_list)-1):\n",
    "        history_list[0].history['acc'] += history_list[i+1].history['acc']\n",
    "        history_list[0].history['val_acc'] += history_list[i+1].history['val_acc']\n",
    "        history_list[0].history['val_loss'] += history_list[i+1].history['val_loss']\n",
    "        history_list[0].history['loss'] += history_list[i+1].history['loss']\n",
    "    return history_list[0]\n",
    "    \n",
    "def to_kaggle_csv(matrix, header,filename):\n",
    "    frame = pd.DataFrame(data = matrix,columns=header)\n",
    "    frame.to_csv(path_or_buf  = filename,index = False,sep =',')\n",
    "    return frame\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "def load_train_valid_test_data(train_file,test_file,validation_percentage):\n",
    "    train_data = np.genfromtxt(delimiter=',',fname=train_file,skip_header=True)\n",
    "    test_data = np.genfromtxt(delimiter=',',fname=test_file,skip_header=True)\n",
    "    train_y = train_data[:,0]\n",
    "    train_x = train_data[:,1:]\n",
    "    train_x = train_x.reshape((train_data.shape[0],28,28,1))\n",
    "    enc = preprocessing.OneHotEncoder()\n",
    "    enc.fit(train_y.reshape((train_y.shape[0],1)))\n",
    "    train_y = enc.transform(train_y.reshape((train_y.shape[0],1))).toarray()\n",
    "    train_x /= 255\n",
    "    test_x = test_data.reshape((test_data.shape[0],28,28,1))\n",
    "    test_x  = test_x/255\n",
    "    train_x = train_x.astype('float32')\n",
    "    test_x = test_x.astype('float32')\n",
    "    train_x, valid_x, train_y, valid_y = train_test_split(train_x, train_y, test_size=0.2, random_state=42)\n",
    "    \n",
    "    return train_x, valid_x, test_x, train_y, valid_y\n",
    "\n",
    "def generate_submission(model,data,filename,file_columns = None):\n",
    "    predictions = model.predict(data)\n",
    "    result = predictions.argmax(axis =1 )\n",
    "    imgs_count = data.shape[0]\n",
    "    result = result.reshape((imgs_count,1))\n",
    "    ids = (np.arange(imgs_count)+1).reshape((imgs_count,1))\n",
    "    result = np.hstack((ids,result))\n",
    "    m = to_kaggle_csv(result,file_columns, filename)\n",
    "\n",
    "def ensemble_submission(models,test_data,filename,file_columns = None):\n",
    "    predictions = model[0].predict(data)\n",
    "    for idx in range(len(models - 1)):\n",
    "        predictions = predictions + model[idx+1].predict(data)\n",
    "    result = predictions.argmax(axis =1)\n",
    "    imgs_count = data.shape[0]\n",
    "    result = result.reshape((imgs_count,1))\n",
    "    ids = (np.arange(imgs_count)+1).reshape((imgs_count,1))\n",
    "    result = np.hstack((ids,result))\n",
    "    m = to_kaggle_csv(result,file_columns, filename)\n",
    "    \n",
    "def validate_preprocessing(train_x, valid_x, test_x, train_y, valid_y):\n",
    "    print('max train',train_x.max())\n",
    "    print('max valid',train_x.max())\n",
    "    print('max test',train_x.max())\n",
    "    \n",
    "    print('train_x shape',train_x.shape)\n",
    "    print('valid_x shape',valid_x.shape)\n",
    "    print('test_x shape',test_x.shape)\n",
    "    print('train_y shape',train_y.shape)\n",
    "    print('valid_y shape',valid_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, valid_x, test_x, train_y, valid_y = load_train_valid_test_data('train.csv','test.csv',0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max train 1.0\n",
      "max valid 1.0\n",
      "max test 1.0\n",
      "train_x shape (33600, 28, 28, 1)\n",
      "valid_x shape (8400, 28, 28, 1)\n",
      "test_x shape (28000, 28, 28, 1)\n",
      "train_y shape (33600, 10)\n",
      "valid_y shape (8400, 10)\n"
     ]
    }
   ],
   "source": [
    "validate_preprocessing(train_x, valid_x, test_x, train_y, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential,Model\n",
    "from keras import optimizers\n",
    "from keras import regularizers\n",
    "from keras.layers.core import Dense, Activation# defining the layers\n",
    "from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Dropout, Flatten\n",
    "from keras.optimizers import RMSprop, SGD , Adam\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K\n",
    "from keras.models import clone_model\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_conv(model,filters):\n",
    "    model.add(Conv2D(filters,(3,3),padding='same'))\n",
    "    model.add(Activation('relu'))\n",
    "    return model\n",
    "    \n",
    "\n",
    "def CNN_architecture(Batch_Norm,dropout):\n",
    "    cnn_model = Sequential()    \n",
    "    cnn_model.add(Conv2D(32,(3,3),padding='same',input_shape=(28,28,1)))\n",
    "    cnn_model.add(Activation('relu'))\n",
    "    cnn_model = add_conv(cnn_model,32)\n",
    "    cnn_model.add(MaxPooling2D(pool_size =(2,2),strides = (2,2)))\n",
    "    cnn_model.add(Dropout(0.2))\n",
    "    cnn_model = add_conv(cnn_model,64)\n",
    "    cnn_model = add_conv(cnn_model,64)\n",
    "    cnn_model.add(MaxPooling2D(pool_size = (2,2),strides = (2,2)))\n",
    "    cnn_model.add(Dropout(0.2))\n",
    "    cnn_model.add(Flatten())\n",
    "    cnn_model.add(Dense(64))\n",
    "    cnn_model.add(Activation('relu'))\n",
    "    if Batch_Norm:\n",
    "        cnn_model.add(BatchNormalization())\n",
    "    cnn_model.add(Dropout(drop_out))\n",
    "    cnn_model.add(Dense(32))\n",
    "    cnn_model.add(Activation('relu'))\n",
    "    if Batch_Norm:\n",
    "        cnn_model.add(BatchNormalization())\n",
    "    cnn_model.add(Dropout(drop_out))\n",
    "    cnn_model.add(Dense(10))\n",
    "    cnn_model.add(Activation('softmax'))\n",
    "    return cnn_model\n",
    "\n",
    "def CNN_model_augmentation(optimizer,epochs,train_x,train_y,valid_x = None,valid_y = None,drop_out = 0.5):\n",
    "    cnn_model = CNN_architecture(True,drop_out)\n",
    "    datagen = ImageDataGenerator( rotation_range=20, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.1,\n",
    "        zoom_range=0.2, fill_mode='nearest')\n",
    "    batch_size = 64\n",
    "    train_generator = datagen.flow(train_x,train_y,batch_size = 64)\n",
    "    steps_per_epoch = int(train_x.shape[0]/batch_size)\n",
    "    optimizer = Adam(lr= 0.001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizer,#RMSprop()\n",
    "                  metrics=['accuracy'])\n",
    "    history = cnn_model.fit_generator(train_generator,epochs=int(epochs*2/3),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y),steps_per_epoch= steps_per_epoch)\n",
    "    optimizer = Adam(lr= 0.0001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizer,#RMSprop()\n",
    "                  metrics=['accuracy'])\n",
    "    history = cnn_model.fit_generator(train_generator,epochs=int(epochs*2/3),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y),steps_per_epoch= steps_per_epoch)\n",
    "    optimizer = Adam(lr= 0.00001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizer,#RMSprop()\n",
    "                  metrics=['accuracy'])\n",
    "    history = cnn_model.fit_generator(train_generator,epochs=int(epochs*2/3),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y),steps_per_epoch= steps_per_epoch)\n",
    "    \n",
    "    return cnn_model,history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Training a convolutional neural network with data augmentation\n",
    "multilayer networks have the capacity to learn features of the input and build upon it in subsequent layers, so it can overfit to the training data, we'll need to make sure it doesn't, that's why we'll be monitoring the training process through both validation and training accuracies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.save?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "drop_outs = [0.3,0.35,0.4,0.45,0.5]\n",
    "models = []\n",
    "histories = []\n",
    "optimizer = 'adam'\n",
    "n_epochs = 15\n",
    "path = 'models/'+str(notebook_num)\n",
    "for drop_out in drop_outs:\n",
    "    model,history = CNN_model_augmentation(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out)\n",
    "    #print(\"Batch Normalization: \", batch)\n",
    "    #print(\"Technique \", technique_dict[tech])\n",
    "    #print(\"drop_out \", drop_out)\n",
    "    models.append(model)\n",
    "    model.save(path + '_'+ str(drop_out)+'_data_augment.h')\n",
    "    histories.append(history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building an ensemble of classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CNN_model_3(optimizer,epochs,train_x,train_y,valid_x = None,valid_y = None,drop_out = 0.5):\n",
    "    cnn_model = CNN_architecture(True,drop_out)\n",
    "    history_list = []\n",
    "    train_generator = data_augmentation_generator(train_x,train_y, batch_size = 64)\n",
    "    batch_size = 64\n",
    "    steps_per_epoch = int(train_x.shape[0]/batch_size)\n",
    "    for i in range(4):\n",
    "        optimizer = Adam(lr= 0.001)\n",
    "        cnn_model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=optimizer,#RMSprop()\n",
    "                      metrics=['accuracy'])\n",
    "        \n",
    "        cnn_model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=optimizer,#RMSprop()\n",
    "                      metrics=['accuracy'])\n",
    "        history = cnn_model.fit_generator(train_generator,epochs=int(epochs*2/3),\n",
    "                            verbose=1,validation_data=(valid_x, valid_y),steps_per_epoch= steps_per_epoch)\n",
    "        history_list.append(history)\n",
    "        optimizer = Adam(lr= 0.0001)\n",
    "        cnn_model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=optimizer,#RMSprop()\n",
    "                      metrics=['accuracy'])\n",
    "        history = cnn_model.fit_generator(train_generator,epochs=int(epochs*2/3),\n",
    "                            verbose=1,validation_data=(valid_x, valid_y),steps_per_epoch= steps_per_epoch)\n",
    "        history_list.append(history)\n",
    "    print('final stage')\n",
    "    print('')\n",
    "    print('')\n",
    "    optimizer = Adam(lr= 0.00001)\n",
    "    cnn_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizer,#RMSprop()\n",
    "                  metrics=['accuracy'])\n",
    "    history = cnn_model.fit_generator(train_generator,epochs=int(epochs*2/3),\n",
    "                        verbose=1,validation_data=(valid_x, valid_y),steps_per_epoch= steps_per_epoch)\n",
    "\n",
    "    history_list.append(history)\n",
    "    return cnn_model, history_from_list(history_list)\n",
    "\n",
    "def CNN_ensemble_models(optimizer,epochs,train_x,train_y,valid_x = None,valid_y = None,drop_out = 0.5):\n",
    "    cnn_model = CNN_architecture(True,drop_out)\n",
    "    history_list = []\n",
    "    for i in range(1):\n",
    "        optimizer = Adam(lr= 0.001)\n",
    "        cnn_model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=optimizer,#RMSprop()\n",
    "                      metrics=['accuracy'])\n",
    "        history = cnn_model.fit(train_x,train_y,\n",
    "                            batch_size = 64,epochs=int(epochs*2/3),\n",
    "                            verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "        history_list.append(history)\n",
    "        optimizer = Adam(lr= 0.0001)\n",
    "        cnn_model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=optimizer,#RMSprop()\n",
    "                      metrics=['accuracy'])\n",
    "        history = cnn_model.fit(train_x,train_y,\n",
    "                            batch_size = 128,epochs=int(epochs*2/3),\n",
    "                            verbose=1,validation_data=(valid_x, valid_y))#,validation_data=(test, Y_test))\n",
    "        history_list.append(history)\n",
    "        models.append(clone_model(cnn_model))\n",
    "    return models,history_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting: 0.45\n",
      "\n",
      "\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 13s 25ms/step - loss: 1.1248 - acc: 0.6376 - val_loss: 0.1557 - val_acc: 0.9735\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.4137 - acc: 0.8790 - val_loss: 0.0761 - val_acc: 0.9751\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.2891 - acc: 0.9173 - val_loss: 0.0534 - val_acc: 0.9833\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.2370 - acc: 0.9351 - val_loss: 0.0398 - val_acc: 0.9868\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 13s 25ms/step - loss: 0.2019 - acc: 0.9449 - val_loss: 0.0406 - val_acc: 0.9871\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1944 - acc: 0.9464 - val_loss: 0.0278 - val_acc: 0.9912\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1723 - acc: 0.9532 - val_loss: 0.0268 - val_acc: 0.9917\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1578 - acc: 0.9568 - val_loss: 0.0335 - val_acc: 0.9904\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1489 - acc: 0.9596 - val_loss: 0.0256 - val_acc: 0.9921\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1499 - acc: 0.9603 - val_loss: 0.0308 - val_acc: 0.9905\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1407 - acc: 0.9619 - val_loss: 0.0285 - val_acc: 0.9914\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1328 - acc: 0.9657 - val_loss: 0.0245 - val_acc: 0.9929\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1300 - acc: 0.9657 - val_loss: 0.0308 - val_acc: 0.9904\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 23s 43ms/step - loss: 0.1096 - acc: 0.9698 - val_loss: 0.0224 - val_acc: 0.9929\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1046 - acc: 0.9733 - val_loss: 0.0192 - val_acc: 0.9939\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 37ms/step - loss: 0.0978 - acc: 0.9739 - val_loss: 0.0193 - val_acc: 0.9938\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1006 - acc: 0.9739 - val_loss: 0.0192 - val_acc: 0.9938\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0934 - acc: 0.9756 - val_loss: 0.0192 - val_acc: 0.9931\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0941 - acc: 0.9758 - val_loss: 0.0190 - val_acc: 0.9936\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0901 - acc: 0.9762 - val_loss: 0.0190 - val_acc: 0.9936\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0903 - acc: 0.9762 - val_loss: 0.0174 - val_acc: 0.9942\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0922 - acc: 0.9758 - val_loss: 0.0181 - val_acc: 0.9939\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0959 - acc: 0.9759 - val_loss: 0.0190 - val_acc: 0.9939\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0860 - acc: 0.9776 - val_loss: 0.0175 - val_acc: 0.9938\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0882 - acc: 0.9774 - val_loss: 0.0171 - val_acc: 0.9940\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0861 - acc: 0.9777 - val_loss: 0.0164 - val_acc: 0.9943\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 22s 42ms/step - loss: 0.1165 - acc: 0.9689 - val_loss: 0.0208 - val_acc: 0.9936\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1153 - acc: 0.9697 - val_loss: 0.0264 - val_acc: 0.9921\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1144 - acc: 0.9707 - val_loss: 0.0397 - val_acc: 0.9893\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1126 - acc: 0.9704 - val_loss: 0.0348 - val_acc: 0.9902\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1139 - acc: 0.9720 - val_loss: 0.0249 - val_acc: 0.9925\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.1042 - acc: 0.9715 - val_loss: 0.0240 - val_acc: 0.9933\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1073 - acc: 0.9729 - val_loss: 0.0303 - val_acc: 0.9915\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0998 - acc: 0.9741 - val_loss: 0.0277 - val_acc: 0.9927\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0978 - acc: 0.9751 - val_loss: 0.0215 - val_acc: 0.9940\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1002 - acc: 0.9745 - val_loss: 0.0203 - val_acc: 0.9946\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0961 - acc: 0.9757 - val_loss: 0.0215 - val_acc: 0.9945\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0911 - acc: 0.9758 - val_loss: 0.0206 - val_acc: 0.9945\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 37ms/step - loss: 0.0990 - acc: 0.9747 - val_loss: 0.0196 - val_acc: 0.9940\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 23s 43ms/step - loss: 0.0826 - acc: 0.9778 - val_loss: 0.0178 - val_acc: 0.9954\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0774 - acc: 0.9802 - val_loss: 0.0186 - val_acc: 0.9950\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0741 - acc: 0.9808 - val_loss: 0.0176 - val_acc: 0.9955\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0720 - acc: 0.9811 - val_loss: 0.0165 - val_acc: 0.9949\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0741 - acc: 0.9817 - val_loss: 0.0168 - val_acc: 0.9945\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0699 - acc: 0.9812 - val_loss: 0.0176 - val_acc: 0.9952\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0704 - acc: 0.9817 - val_loss: 0.0165 - val_acc: 0.9957\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0692 - acc: 0.9832 - val_loss: 0.0173 - val_acc: 0.9952\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0706 - acc: 0.9823 - val_loss: 0.0155 - val_acc: 0.9954\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0659 - acc: 0.9819 - val_loss: 0.0165 - val_acc: 0.9954\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 19s 37ms/step - loss: 0.0687 - acc: 0.9830 - val_loss: 0.0167 - val_acc: 0.9950\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0665 - acc: 0.9831 - val_loss: 0.0169 - val_acc: 0.9950\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0657 - acc: 0.9825 - val_loss: 0.0171 - val_acc: 0.9954\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 23s 44ms/step - loss: 0.0900 - acc: 0.9767 - val_loss: 0.0196 - val_acc: 0.9949\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0861 - acc: 0.9783 - val_loss: 0.0232 - val_acc: 0.9927\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0916 - acc: 0.9766 - val_loss: 0.0227 - val_acc: 0.9938\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0888 - acc: 0.9769 - val_loss: 0.0171 - val_acc: 0.9952\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0796 - acc: 0.9792 - val_loss: 0.0199 - val_acc: 0.9944\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0789 - acc: 0.9795 - val_loss: 0.0225 - val_acc: 0.9939\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0898 - acc: 0.9763 - val_loss: 0.0181 - val_acc: 0.9951\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0820 - acc: 0.9786 - val_loss: 0.0236 - val_acc: 0.9937\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 19s 37ms/step - loss: 0.0851 - acc: 0.9773 - val_loss: 0.0206 - val_acc: 0.9949\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0849 - acc: 0.9783 - val_loss: 0.0170 - val_acc: 0.9952\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0772 - acc: 0.9809 - val_loss: 0.0213 - val_acc: 0.9939\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0788 - acc: 0.9799 - val_loss: 0.0182 - val_acc: 0.9940\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0803 - acc: 0.9782 - val_loss: 0.0185 - val_acc: 0.9946\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 23s 43ms/step - loss: 0.0706 - acc: 0.9821 - val_loss: 0.0173 - val_acc: 0.9951\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0635 - acc: 0.9842 - val_loss: 0.0156 - val_acc: 0.9958\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0637 - acc: 0.9838 - val_loss: 0.0170 - val_acc: 0.9958\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0616 - acc: 0.9837 - val_loss: 0.0157 - val_acc: 0.9958\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0642 - acc: 0.9838 - val_loss: 0.0164 - val_acc: 0.9956\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0609 - acc: 0.9841 - val_loss: 0.0158 - val_acc: 0.9956\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0608 - acc: 0.9853 - val_loss: 0.0166 - val_acc: 0.9957\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0612 - acc: 0.9847 - val_loss: 0.0160 - val_acc: 0.9956\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 21s 40ms/step - loss: 0.0557 - acc: 0.9860 - val_loss: 0.0155 - val_acc: 0.9954\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0584 - acc: 0.9851 - val_loss: 0.0166 - val_acc: 0.9951\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0581 - acc: 0.9850 - val_loss: 0.0164 - val_acc: 0.9956\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0617 - acc: 0.9840 - val_loss: 0.0158 - val_acc: 0.9957\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0612 - acc: 0.9844 - val_loss: 0.0155 - val_acc: 0.9958\n",
      "final stage\n",
      "\n",
      "\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 24s 45ms/step - loss: 0.0564 - acc: 0.9857 - val_loss: 0.0158 - val_acc: 0.9958\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0581 - acc: 0.9855 - val_loss: 0.0157 - val_acc: 0.9956\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0594 - acc: 0.9851 - val_loss: 0.0158 - val_acc: 0.9958\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0587 - acc: 0.9855 - val_loss: 0.0155 - val_acc: 0.9957\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0591 - acc: 0.9846 - val_loss: 0.0158 - val_acc: 0.9956\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0580 - acc: 0.9850 - val_loss: 0.0157 - val_acc: 0.9956\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0578 - acc: 0.9847 - val_loss: 0.0157 - val_acc: 0.9956\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0594 - acc: 0.9851 - val_loss: 0.0156 - val_acc: 0.9956\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0537 - acc: 0.9866 - val_loss: 0.0156 - val_acc: 0.9957\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0561 - acc: 0.9857 - val_loss: 0.0157 - val_acc: 0.9954\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0558 - acc: 0.9856 - val_loss: 0.0158 - val_acc: 0.9957\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0536 - acc: 0.9866 - val_loss: 0.0156 - val_acc: 0.9956\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0534 - acc: 0.9868 - val_loss: 0.0157 - val_acc: 0.9955\n",
      "starting: 0.44\n",
      "\n",
      "\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 24s 46ms/step - loss: 1.2301 - acc: 0.6012 - val_loss: 0.1267 - val_acc: 0.9720\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.4488 - acc: 0.8694 - val_loss: 0.0859 - val_acc: 0.9769\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.3138 - acc: 0.9119 - val_loss: 0.0555 - val_acc: 0.9825\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.2519 - acc: 0.9292 - val_loss: 0.0406 - val_acc: 0.9874\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.2130 - acc: 0.9404 - val_loss: 0.0396 - val_acc: 0.9873\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 21s 40ms/step - loss: 0.1945 - acc: 0.9448 - val_loss: 0.0495 - val_acc: 0.9861\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 21s 40ms/step - loss: 0.1840 - acc: 0.9497 - val_loss: 0.0304 - val_acc: 0.9900\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 21s 40ms/step - loss: 0.1721 - acc: 0.9518 - val_loss: 0.0361 - val_acc: 0.9901\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.1515 - acc: 0.9591 - val_loss: 0.0307 - val_acc: 0.9904\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1548 - acc: 0.9587 - val_loss: 0.0235 - val_acc: 0.9926\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 21s 40ms/step - loss: 0.1374 - acc: 0.9633 - val_loss: 0.0236 - val_acc: 0.9918\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.1373 - acc: 0.9635 - val_loss: 0.0311 - val_acc: 0.9904\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.1369 - acc: 0.9643 - val_loss: 0.0267 - val_acc: 0.9924\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 15s 29ms/step - loss: 0.1121 - acc: 0.9711 - val_loss: 0.0213 - val_acc: 0.9932\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1045 - acc: 0.9720 - val_loss: 0.0202 - val_acc: 0.9942\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0959 - acc: 0.9743 - val_loss: 0.0206 - val_acc: 0.9936\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0997 - acc: 0.9736 - val_loss: 0.0194 - val_acc: 0.9940\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0937 - acc: 0.9754 - val_loss: 0.0179 - val_acc: 0.9949\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0930 - acc: 0.9760 - val_loss: 0.0201 - val_acc: 0.9939\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0944 - acc: 0.9765 - val_loss: 0.0188 - val_acc: 0.9945\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0939 - acc: 0.9761 - val_loss: 0.0180 - val_acc: 0.9936\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0922 - acc: 0.9762 - val_loss: 0.0203 - val_acc: 0.9933\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0904 - acc: 0.9767 - val_loss: 0.0186 - val_acc: 0.9944\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0898 - acc: 0.9769 - val_loss: 0.0189 - val_acc: 0.9944\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0889 - acc: 0.9779 - val_loss: 0.0186 - val_acc: 0.9945\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0907 - acc: 0.9763 - val_loss: 0.0182 - val_acc: 0.9951\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 13s 26ms/step - loss: 0.1175 - acc: 0.9686 - val_loss: 0.0237 - val_acc: 0.9931\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1161 - acc: 0.9688 - val_loss: 0.0417 - val_acc: 0.9877\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1150 - acc: 0.9701 - val_loss: 0.0242 - val_acc: 0.9924\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1096 - acc: 0.9708 - val_loss: 0.0232 - val_acc: 0.9933\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1088 - acc: 0.9719 - val_loss: 0.0290 - val_acc: 0.9912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1066 - acc: 0.9729 - val_loss: 0.0227 - val_acc: 0.9932\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1061 - acc: 0.9725 - val_loss: 0.0174 - val_acc: 0.9940\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0994 - acc: 0.9735 - val_loss: 0.0238 - val_acc: 0.9929\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0996 - acc: 0.9727 - val_loss: 0.0304 - val_acc: 0.9919\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0966 - acc: 0.9744 - val_loss: 0.0254 - val_acc: 0.9933\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0976 - acc: 0.9739 - val_loss: 0.0190 - val_acc: 0.9939\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0960 - acc: 0.9758 - val_loss: 0.0206 - val_acc: 0.9938\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0941 - acc: 0.9750 - val_loss: 0.0191 - val_acc: 0.9942\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 14s 26ms/step - loss: 0.0822 - acc: 0.9786 - val_loss: 0.0165 - val_acc: 0.9949\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0779 - acc: 0.9792 - val_loss: 0.0178 - val_acc: 0.9948\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0783 - acc: 0.9795 - val_loss: 0.0165 - val_acc: 0.9954\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0702 - acc: 0.9819 - val_loss: 0.0166 - val_acc: 0.9946\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0751 - acc: 0.9820 - val_loss: 0.0162 - val_acc: 0.9948\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0700 - acc: 0.9824 - val_loss: 0.0155 - val_acc: 0.9951\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0682 - acc: 0.9827 - val_loss: 0.0167 - val_acc: 0.9948\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0704 - acc: 0.9814 - val_loss: 0.0164 - val_acc: 0.9951\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0666 - acc: 0.9828 - val_loss: 0.0156 - val_acc: 0.9949\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0750 - acc: 0.9809 - val_loss: 0.0164 - val_acc: 0.9948\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0683 - acc: 0.9831 - val_loss: 0.0163 - val_acc: 0.9951\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0697 - acc: 0.9820 - val_loss: 0.0163 - val_acc: 0.9955\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0692 - acc: 0.9829 - val_loss: 0.0166 - val_acc: 0.9950\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 14s 27ms/step - loss: 0.0826 - acc: 0.9785 - val_loss: 0.0194 - val_acc: 0.9945\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0866 - acc: 0.9776 - val_loss: 0.0282 - val_acc: 0.9924\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0896 - acc: 0.9766 - val_loss: 0.0256 - val_acc: 0.9921\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0877 - acc: 0.9781 - val_loss: 0.0218 - val_acc: 0.9933\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0846 - acc: 0.9794 - val_loss: 0.0195 - val_acc: 0.9933\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0808 - acc: 0.9774 - val_loss: 0.0193 - val_acc: 0.9940\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0892 - acc: 0.9760 - val_loss: 0.0175 - val_acc: 0.9944\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0849 - acc: 0.9793 - val_loss: 0.0230 - val_acc: 0.9938\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0824 - acc: 0.9786 - val_loss: 0.0211 - val_acc: 0.9936\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0824 - acc: 0.9785 - val_loss: 0.0217 - val_acc: 0.9929\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0833 - acc: 0.9793 - val_loss: 0.0214 - val_acc: 0.9940\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0773 - acc: 0.9799 - val_loss: 0.0252 - val_acc: 0.9920\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0780 - acc: 0.9803 - val_loss: 0.0219 - val_acc: 0.9937\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 14s 26ms/step - loss: 0.0671 - acc: 0.9824 - val_loss: 0.0186 - val_acc: 0.9939\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0629 - acc: 0.9837 - val_loss: 0.0166 - val_acc: 0.9943\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0660 - acc: 0.9832 - val_loss: 0.0173 - val_acc: 0.9940\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0642 - acc: 0.9833 - val_loss: 0.0165 - val_acc: 0.9944\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0662 - acc: 0.9835 - val_loss: 0.0172 - val_acc: 0.9945\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0630 - acc: 0.9846 - val_loss: 0.0165 - val_acc: 0.9943\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0616 - acc: 0.9838 - val_loss: 0.0163 - val_acc: 0.9940\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0615 - acc: 0.9845 - val_loss: 0.0154 - val_acc: 0.9945\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0592 - acc: 0.9848 - val_loss: 0.0158 - val_acc: 0.9945\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0568 - acc: 0.9863 - val_loss: 0.0160 - val_acc: 0.9946\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0574 - acc: 0.9856 - val_loss: 0.0152 - val_acc: 0.9952\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0573 - acc: 0.9854 - val_loss: 0.0161 - val_acc: 0.9946\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0585 - acc: 0.9850 - val_loss: 0.0166 - val_acc: 0.9948\n",
      "final stage\n",
      "\n",
      "\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 14s 26ms/step - loss: 0.0567 - acc: 0.9854 - val_loss: 0.0165 - val_acc: 0.9944\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 18s 35ms/step - loss: 0.0626 - acc: 0.9847 - val_loss: 0.0163 - val_acc: 0.9945\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0563 - acc: 0.9857 - val_loss: 0.0161 - val_acc: 0.9946\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0572 - acc: 0.9854 - val_loss: 0.0163 - val_acc: 0.9945\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0588 - acc: 0.9853 - val_loss: 0.0161 - val_acc: 0.9945\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0552 - acc: 0.9854 - val_loss: 0.0163 - val_acc: 0.9944\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0556 - acc: 0.9857 - val_loss: 0.0163 - val_acc: 0.9945\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0625 - acc: 0.9847 - val_loss: 0.0162 - val_acc: 0.9945\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0535 - acc: 0.9865 - val_loss: 0.0160 - val_acc: 0.9946\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0549 - acc: 0.9859 - val_loss: 0.0161 - val_acc: 0.9945\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 37ms/step - loss: 0.0573 - acc: 0.9861 - val_loss: 0.0159 - val_acc: 0.9945\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 12s 23ms/step - loss: 0.0546 - acc: 0.9863 - val_loss: 0.0160 - val_acc: 0.9945\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0548 - acc: 0.9868 - val_loss: 0.0161 - val_acc: 0.9945\n",
      "starting: 0.45\n",
      "\n",
      "\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 14s 27ms/step - loss: 1.1456 - acc: 0.6280 - val_loss: 0.1660 - val_acc: 0.9670\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.4483 - acc: 0.8687 - val_loss: 0.0727 - val_acc: 0.9776\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.3129 - acc: 0.9115 - val_loss: 0.0494 - val_acc: 0.9838\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.2500 - acc: 0.9290 - val_loss: 0.0520 - val_acc: 0.9836\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 12s 22ms/step - loss: 0.2180 - acc: 0.9379 - val_loss: 0.0383 - val_acc: 0.9876\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 12s 23ms/step - loss: 0.1970 - acc: 0.9471 - val_loss: 0.0392 - val_acc: 0.9883\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.1819 - acc: 0.9498 - val_loss: 0.0307 - val_acc: 0.9902\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.1678 - acc: 0.9546 - val_loss: 0.0310 - val_acc: 0.9910\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.1609 - acc: 0.9560 - val_loss: 0.0245 - val_acc: 0.9917\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 12s 23ms/step - loss: 0.1438 - acc: 0.9601 - val_loss: 0.0247 - val_acc: 0.9923\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.1416 - acc: 0.9617 - val_loss: 0.0262 - val_acc: 0.9919\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.1372 - acc: 0.9643 - val_loss: 0.0193 - val_acc: 0.9926\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.1291 - acc: 0.9659 - val_loss: 0.0335 - val_acc: 0.9898\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 15s 29ms/step - loss: 0.1132 - acc: 0.9707 - val_loss: 0.0217 - val_acc: 0.9930\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1040 - acc: 0.9720 - val_loss: 0.0185 - val_acc: 0.9940\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1031 - acc: 0.9726 - val_loss: 0.0202 - val_acc: 0.9935\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1016 - acc: 0.9747 - val_loss: 0.0198 - val_acc: 0.9938\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0974 - acc: 0.9748 - val_loss: 0.0185 - val_acc: 0.9943\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0946 - acc: 0.9763 - val_loss: 0.0187 - val_acc: 0.9940\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 19s 37ms/step - loss: 0.0941 - acc: 0.9749 - val_loss: 0.0186 - val_acc: 0.9937\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0906 - acc: 0.9766 - val_loss: 0.0183 - val_acc: 0.9942\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.0915 - acc: 0.9765 - val_loss: 0.0185 - val_acc: 0.9946\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 13s 24ms/step - loss: 0.0923 - acc: 0.9748 - val_loss: 0.0184 - val_acc: 0.9946\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 13s 25ms/step - loss: 0.0898 - acc: 0.9764 - val_loss: 0.0171 - val_acc: 0.9946\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 12s 24ms/step - loss: 0.0869 - acc: 0.9773 - val_loss: 0.0170 - val_acc: 0.9946\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 12s 22ms/step - loss: 0.0860 - acc: 0.9784 - val_loss: 0.0177 - val_acc: 0.9942\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 14s 27ms/step - loss: 0.1184 - acc: 0.9704 - val_loss: 0.0348 - val_acc: 0.9907\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1265 - acc: 0.9653 - val_loss: 0.0253 - val_acc: 0.9915\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1143 - acc: 0.9698 - val_loss: 0.0266 - val_acc: 0.9926\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1127 - acc: 0.9705 - val_loss: 0.0238 - val_acc: 0.9935\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.1105 - acc: 0.9715 - val_loss: 0.0208 - val_acc: 0.9935\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1110 - acc: 0.9698 - val_loss: 0.0188 - val_acc: 0.9936\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1046 - acc: 0.9723 - val_loss: 0.0250 - val_acc: 0.9927\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1053 - acc: 0.9726 - val_loss: 0.0208 - val_acc: 0.9939\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0950 - acc: 0.9738 - val_loss: 0.0199 - val_acc: 0.9938\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1020 - acc: 0.9742 - val_loss: 0.0213 - val_acc: 0.9925\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.1066 - acc: 0.9721 - val_loss: 0.0218 - val_acc: 0.9932\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0937 - acc: 0.9754 - val_loss: 0.0200 - val_acc: 0.9932\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 11s 22ms/step - loss: 0.0968 - acc: 0.9749 - val_loss: 0.0215 - val_acc: 0.9933\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 14s 28ms/step - loss: 0.0842 - acc: 0.9781 - val_loss: 0.0162 - val_acc: 0.9948\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 19s 37ms/step - loss: 0.0784 - acc: 0.9797 - val_loss: 0.0165 - val_acc: 0.9946\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 19s 37ms/step - loss: 0.0782 - acc: 0.9802 - val_loss: 0.0161 - val_acc: 0.9950\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 19s 36ms/step - loss: 0.0723 - acc: 0.9806 - val_loss: 0.0166 - val_acc: 0.9950\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 15s 29ms/step - loss: 0.0713 - acc: 0.9809 - val_loss: 0.0156 - val_acc: 0.9955\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0737 - acc: 0.9813 - val_loss: 0.0175 - val_acc: 0.9943\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 11s 21ms/step - loss: 0.0745 - acc: 0.9808 - val_loss: 0.0162 - val_acc: 0.9954\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 17s 32ms/step - loss: 0.0734 - acc: 0.9808 - val_loss: 0.0167 - val_acc: 0.9951\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0687 - acc: 0.9821 - val_loss: 0.0149 - val_acc: 0.9954\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0717 - acc: 0.9814 - val_loss: 0.0164 - val_acc: 0.9955\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0687 - acc: 0.9818 - val_loss: 0.0167 - val_acc: 0.9952\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0676 - acc: 0.9835 - val_loss: 0.0168 - val_acc: 0.9950\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0696 - acc: 0.9824 - val_loss: 0.0156 - val_acc: 0.9956\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 24s 46ms/step - loss: 0.0874 - acc: 0.9772 - val_loss: 0.0219 - val_acc: 0.9939\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0912 - acc: 0.9762 - val_loss: 0.0198 - val_acc: 0.9943\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0881 - acc: 0.9766 - val_loss: 0.0239 - val_acc: 0.9932\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0904 - acc: 0.9766 - val_loss: 0.0213 - val_acc: 0.9943\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0848 - acc: 0.9779 - val_loss: 0.0204 - val_acc: 0.9938\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0829 - acc: 0.9782 - val_loss: 0.0211 - val_acc: 0.9937\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 20s 37ms/step - loss: 0.0807 - acc: 0.9790 - val_loss: 0.0184 - val_acc: 0.9951\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 38ms/step - loss: 0.0865 - acc: 0.9773 - val_loss: 0.0177 - val_acc: 0.9951\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0840 - acc: 0.9783 - val_loss: 0.0233 - val_acc: 0.9938\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0855 - acc: 0.9783 - val_loss: 0.0326 - val_acc: 0.9913\n",
      "Epoch 11/13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0806 - acc: 0.9787 - val_loss: 0.0224 - val_acc: 0.9939\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0797 - acc: 0.9801 - val_loss: 0.0215 - val_acc: 0.9943\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0774 - acc: 0.9796 - val_loss: 0.0221 - val_acc: 0.9943\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 24s 46ms/step - loss: 0.0705 - acc: 0.9826 - val_loss: 0.0176 - val_acc: 0.9948\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0652 - acc: 0.9832 - val_loss: 0.0178 - val_acc: 0.9950\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0641 - acc: 0.9831 - val_loss: 0.0165 - val_acc: 0.9950\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0638 - acc: 0.9842 - val_loss: 0.0164 - val_acc: 0.9952\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0645 - acc: 0.9839 - val_loss: 0.0161 - val_acc: 0.9952\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0614 - acc: 0.9845 - val_loss: 0.0160 - val_acc: 0.9956\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 19s 37ms/step - loss: 0.0589 - acc: 0.9850 - val_loss: 0.0170 - val_acc: 0.9952\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0629 - acc: 0.9843 - val_loss: 0.0158 - val_acc: 0.9952\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0604 - acc: 0.9851 - val_loss: 0.0163 - val_acc: 0.9955\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0609 - acc: 0.9838 - val_loss: 0.0165 - val_acc: 0.9955\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0590 - acc: 0.9848 - val_loss: 0.0165 - val_acc: 0.9954\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0570 - acc: 0.9852 - val_loss: 0.0163 - val_acc: 0.9952\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0605 - acc: 0.9846 - val_loss: 0.0159 - val_acc: 0.9955\n",
      "final stage\n",
      "\n",
      "\n",
      "Epoch 1/13\n",
      "525/525 [==============================] - 24s 46ms/step - loss: 0.0625 - acc: 0.9840 - val_loss: 0.0158 - val_acc: 0.9957\n",
      "Epoch 2/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0592 - acc: 0.9851 - val_loss: 0.0158 - val_acc: 0.9956\n",
      "Epoch 3/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0583 - acc: 0.9850 - val_loss: 0.0158 - val_acc: 0.9957\n",
      "Epoch 4/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0590 - acc: 0.9851 - val_loss: 0.0158 - val_acc: 0.9960\n",
      "Epoch 5/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0574 - acc: 0.9850 - val_loss: 0.0157 - val_acc: 0.9958\n",
      "Epoch 6/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0599 - acc: 0.9845 - val_loss: 0.0156 - val_acc: 0.9958\n",
      "Epoch 7/13\n",
      "525/525 [==============================] - 19s 36ms/step - loss: 0.0610 - acc: 0.9841 - val_loss: 0.0159 - val_acc: 0.9957\n",
      "Epoch 8/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0608 - acc: 0.9841 - val_loss: 0.0159 - val_acc: 0.9957\n",
      "Epoch 9/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0577 - acc: 0.9857 - val_loss: 0.0160 - val_acc: 0.9958\n",
      "Epoch 10/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0605 - acc: 0.9849 - val_loss: 0.0160 - val_acc: 0.9957\n",
      "Epoch 11/13\n",
      "525/525 [==============================] - 20s 39ms/step - loss: 0.0550 - acc: 0.9866 - val_loss: 0.0160 - val_acc: 0.9957\n",
      "Epoch 12/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0544 - acc: 0.9854 - val_loss: 0.0160 - val_acc: 0.9957\n",
      "Epoch 13/13\n",
      "525/525 [==============================] - 21s 39ms/step - loss: 0.0532 - acc: 0.9858 - val_loss: 0.0161 - val_acc: 0.9957\n"
     ]
    }
   ],
   "source": [
    "drop_outs = [0.45,0.44,0.45]#,0.5,0.55]\n",
    "ensemble_models = []\n",
    "ensemble_histories = []\n",
    "optimizer = 'adam'\n",
    "n_epochs = 20\n",
    "for idx,drop_out in enumerate(drop_outs):\n",
    "    print(\"starting:\", drop_outs[idx])\n",
    "    print(\"\");print(\"\")\n",
    "    model,history = CNN_model_3(optimizer,n_epochs, train_x, train_y, valid_x, valid_y,drop_out = drop_out)\n",
    "    #print(\"Batch Normalization: \", batch)\n",
    "    #print(\"Technique \", technique_dict[tech])\n",
    "    #print(\"drop_out \", drop_out)\n",
    "    model.save(path + '_'+ str(drop_out)+'_ensemble_' + str(idx)+'.h')\n",
    "    ensemble_models.append(clone_model(model))\n",
    "    ensemble_histories.append(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "clone_model?\n",
    "val_acc_list = []\n",
    "print('validation accuracies for different dropouts')\n",
    "for history in ensemble_histories:\n",
    "    \n",
    "    print(history.history['val_acc'][-1])\n",
    "    val_acc_list.append(history.history['val_acc'])\n",
    "val_acc_np = np.array(val_acc_list)\n",
    "arg = np.argmax(val_acc_np,axis = 0)[-1]\n",
    "print (arg,val_acc_np[3,-1])\n",
    "models[3].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/4_0.45_ensemble_0.h\n",
      "model acc 0.995476190476\n",
      "models/4_0.44_ensemble_1.h\n",
      "model acc 0.994523809524\n",
      "models/4_0.45_ensemble_2.h\n",
      "model acc 0.995714285714\n",
      "ensemble acc 0.995833333333\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "models = []\n",
    "drop_outs = [0.45,0.44,0.45]\n",
    "for idx,drop_out in enumerate(drop_outs):\n",
    "    print(path + '_'+ str(drop_out)+'_ensemble_' + str(idx)+'.h')\n",
    "    model = load_model(path + '_'+ str(drop_out)+'_ensemble_' + str(idx)+'.h')\n",
    "    predictions = model.predict(valid_x)\n",
    "    #print(predictions[0,0])\n",
    "    #results = np.mean(results,axis = 0)\n",
    "    results.append(predictions)\n",
    "    predictions = np.argmax(predictions,axis = 1)\n",
    "    g_truth = np.argmax(valid_y,axis = 1)\n",
    "    #print(predictions.shape,g_truth)\n",
    "    print('model acc',np.sum(predictions == g_truth)*1.0/predictions.shape[0])\n",
    "    \n",
    "    models.append(clone_model(model))\n",
    "results = np.array(results)\n",
    "results = np.mean(results,axis = 0)\n",
    "predictions = np.argmax(results,axis = 1)\n",
    "g_truth = np.argmax(valid_y,axis = 1)\n",
    "print('ensemble acc',np.sum(predictions == g_truth)*1.0/predictions.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "## best model submission\n",
    "model = load_model(path + '_'+ str(0.45)+'_ensemble_' + str(2)+'.h')\n",
    "generate_submission(model,test_x,'4-sub_adam_cnn_2c_1p_2c_1p_valid9954_batch_lrannealing.csv',['ImageId','Label'])"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAABCYAAACUCAYAAABP5ihSAAAgAElEQVR4Ae2dC3BV13nv/wIEOMJCGIuHghQeCg+DbYTaW0RrIiC2wanBdzpAOhNjwZAZ10AhSTH2OG2Utr4OkAfExsmdcAHbufcG6KRGmfJyMMS5NWpaELEhCAIGS1g8BBbICDAYdOdbe6991t5n7/OQzjk6R/prRnP2Xnutb33rt9/f/r5vZV379GYrAv7ONTQgL69vwFYWkwAJkAAJkAAJkAAJkAAJkAAJkAAJkICbQL+8PJw6/SEGFRS4NwSsZWHVrwMNEwFtWEwCJEACJEACJEACJEACJEACJEACJEAC7SJwbelk1b5bu6SwMQmQAAmQAAmQAAmQAAmQAAmQAAmQAAm0g0AP3fbkwol60fm9cO4cigqHOOuycPv2bdc6V0iABEiABEiABEiABEiABEiABEiABDqGQPfu3Tum4yi91tWfwYBBg3xrjVhf7Sqnx4QLB1dIgARIgARIgARIgARIgARIgARIgARSSYCGiVTSZl8kQAIkQAIkQAIkQAIkQAIkQAIkQAIuAjRMuHBwhQRIgARIgARIgARIgARIgARIgARIIJUEnBwTqeyUfZEACZAACZAACZAACZAACbgJ3LlzB1evXsVnn32G1tbkTJyXlZWFHj16oE+fPujWjd8o3XuAayRAAh1FgFejjiLPfkmABEiABEiABEiABEjAIPDJJ5/g1q1bSTNKSFdi8JA+pC/+kQAJkEC6EKBhIl32BPUgARIgARIgARIgARLo0gRSOftdKvvq0juVgycBEoiJAA0TMWFiJRIgARIgARIgARIgARJILoFkhW/4aZ3Kvvz6ZxkJkAAJmASYY8KkwWUSIAESIAESIAESIAESSHMCYlSQPBTyL2EZkpuiZ8+eKmeE/k3zIVA9EiABEnARoGHChYMrJEACJEACJEACJEACJJC+BMQo0dTUBEliKeEYffv2xaVLl3D+/HncvHlTGSnuu+8+ZGdnp+8gqBkJkAAJeAgwlMMDhKskQAIkQAIkQAIkQAIkkK4EdILML3zhCxg2bBiuX7+Oa9eu4Z577kFhYaHyovjjH/+YrupTLxIgARLwJUCPCV8sLCQBEiABEiABEiABEiCB9CEgnhKNjY1oaGhQxggJ45C/999/X00xev/99yvviV69emH//v0YNWoUunfvnj4DoCYkQAIkEIEAPSYiwOEmEiABEiABEiABEiABEuhIAmKQkJCNc+fOqd/y8nIMGjQIZ8+eVVN+fvGLX8SYMWMwfPhw9O/fX+Wc6NatG06fPp3UaUc7kgn7JgES6HwE6DHR+fYpR0QCJEACJEACJEACJNBJCGgPCTE2yL/kkThz5owySkgox8CBA5UBQnJOSOLLwYMH49NPP8WJEyfw+c9/Hr179+4kJDgMEiCBzkwgIw0TzTXr8M11Nc5+mV25Ho8OcVathTO7sLByq1PoW6e5Buu+uQ6OpNmVWB8myBHRiRaa8M6eedh1Uw9pJr424+sYo1f178mf4YXjVXoN+YNfx7Lx/Zz1zFloRNXCUizebWq8AtvqFqHELErLZUP3R17BgfUzkZ+WelIpEiABEiABEiCBRBOorq5GaWmp8pA4cuQICgoKVNjGpEmT8Mtf/hIStiEzckgCTAntkNANWb/rrrtUMkxJiinGCf4BqH0dT760z0Ex5fkNWDDaWQ1cqN24AC86zYZj/g++jan3uqtf3PvP+MamD5zCcNnHsOGpldjr1NAL5XjhtXnwVcPWt7hiFb4zxd1h9P60/K73K/tr89BwZopEG4+B9KX4Kc5UV6H5vtm43/OK9mn9fmzbX2+rXoiyWWUo7JW+IxHNMs4wYRklilG5fhGULUIZIBYCpnFClZ3Aoh+uR0kugMA6WyEGi0VKUDNq1n0TC9HZjRO2UeJz38eL00apo/Po/sfx8x1wGScaD30Ha84Owddm/Mo2WBzDz3fMwxpkmHGiZh2KZq0EYBoiarCuaBZmFa3Eim11WJTO1onG/ajSBpXdVdjfOBMzE2iZaKxaiNLFu/HIKwewPpGC0/u6R+1IgARIgARIIO0JXLx4EQMGDFB5JSRnhCS9lHwS4hkh2yTE4/jx4+jXr5/ylBDPCDFUyJ8YJsS74sMPP6RhQoCoF9I6zP/BBsuooNYXABGNExfx9j8+i42Yhx+9Vg5lGpB23/pnwDBOWIYLMTB82zIwXNyH735rATaEyfY3avgfiMewwTCimHUso0RRqL+YxmJK6MTLF/dhsxiRKnzG2KZjwEdOOhU1n4TYHsbe51bKMkrkYvqcOZBXYTQfwZZtW9A8fQ7GqgJ3/XRZy7AcE2ewf10NShaVWUYJoTjkUVTOBrZuq0GzotqMmm1bgdmzLKNExDqVhqdFLkqeXISSrdtQYwlKl32UWD1O/gt23ZyJr5VZRgkRPmbUMuSjCgdO6q6OYdfZgxgz0vSiGIWvjZyJxrP/gqO6Wtr/1mCdMko8glcOmN4RJVh04BU8AmDlLMNjJg3H07i/CrvxCFasEG13o2p/YxpqSZVIgARIgARIgAQSTUBm2RCjg/z++Z//Ob785S8rw8Pu3bshs26I0aGmpga1tbX46KOPVPjGlStXlBpixJA//Zto3TJL3kW8vWUfUD475Okweh5eKAf2btmHi0GDqd2OjSeHY/4ztlFC6ql2H2Djr45ZrewX4SnPG14P95ZjUcVwt+yLZ6G/XQd1Z5Zf3LsVe8vnYf4Is1SWj6Fq0wcorngs5GURy1i8YjrbuhiDnlqAJ7/1Ok74jq2Nx4CvrDQobP4Dtm7diq27jvgo04wT++sxdvpYyyghNXLHYvpY4MiRenzq0yJdijLLMNHciBMowaQRkUw9zbhQA8we647tGDJ2NlBzwTFeSJ2SAR45ufkoRg3ePdl5LRONn5wBeg63rL76KLxniFq/+EmTVfLxGVzEBBR5XILQb7jHgKEFpOdvY9U6iK8EViwK9zLIn4lFK2TjSqyrSteX/UbsF3eJR2ZizpyZypCyu2o/Eqlt/sz1qKuro7dEeh7C1IoESIAESKALExCPBzFMiCeEhGiIN8TkyZPx3HPP4Utf+pKadePq1av4wx/+oAwV//7v/46qqirs2bNH5aHQ+Si6MEJ76Jdw+iQwxfgoJxtGl5UDJ+sCDRMXz9YBKEKBO4oC9w4dDuz7D9SKkIt1OIHhGOqtM7gIOPn/8F6g1SPCXrm4D+s2AfMfHxdeSRk4huOh+z0dhtfsWiX3luM7r23AG6+t8jHmCIq2HQNpCzH3PsyePRuzZ5ah0Ktkcz2OYCwKva+5hWOB+jO4kMaWicwyTOSWYNH6RSFPCLUjmtEoprHifNtVxTJeeG0OyB2AEpxAo7I55GJAidgpPAYIZfjw7t1Otl60DC9Om+bOU6AMEcC9d5uWiIOos+0UDoGmDxL6UuzITdLCmXorBuKRQreRSnc3pFC8EIDd9WfsIsnnUISiooWoqqnCwiJZtv6tlCZ6u1W20GvQkLARo42SY1oR7O3STkIopK6RKkWrFfq1wzgemVmG/PwyzFROExLOEaoiS1rWwqoaW39bb1u43q50W1jl2od6W2gsEuYi7dehxjOeiLq6VeIaCZAACZAACZBAkgiIB8VDDz2Exx9/HPPmzUNubq7KJ/G5z30OOTk5uHbtmjJWnD9/Hi0tLUnSIoPE2i/zXuMB7i1CMerQEGA8uFeMCz7bL54O5ZKwZHyA0x4ZllHDy6gIeP+f8aR82Vf/r1vGDVe1i3j71deBiqdD3h3mdvUC7s1xcREN4o5RONj94dFs19WX23gMZCK2T680Q6wSYekkeuWiEPVoNgwTzUe2YMuW0P8Rz6txqsefWYYJHzrNNW9gXU0JFpXZL5/NF0LJLMPq18CyReRixKQSYGsldul3UjSj5o30dusPG44uaDqITW/9LX5x0slmqbeE/ebfYxofZHMT3qlZg8aey/Codhe7ZxoezQOOHv+ZEbZxDD+3E2E6nhVh0tOpoBH1TlbTKHrV1Lte1iVkYvE64KW6OtTVbYM4VqyctRALFz4vhairO4BXHgF2L34ejm2isQoLJWxEElSqdlJnNxaX+hxTVc+jdHEJttVFzm+hwzhmlklSiXyUWZaJwHCO3ZbSygNim6U0Fi5ciOfxkio7YCmN5x2lI3FZiVnvTlLtQgx8xhJJBLeRAAmQAAmQAAkkhYDkmejRo4fyqCguLsaDDz6oZuOQRJl6+lDxtpCEmF3+T3k1BFEINyo4NUf/GabgA2x81Qj3qH3dSIQJ4N5xeGgEsPclw8igPB4M44UIVDrsw2/xNN5QX/Y34IXyfXjxKaOd1JPwEczDIk+yS0cnn4WLe39qhZw8HgrT9qnWtYvaegxkILVPm2MLGpI8FDuPjFV5KObMmYM508fiyM4jdnRBxww8sw0TZ3ap2TlmV3q9KKLDzC1ZhPWVs7G1Ul445f+buDDrh+mdCNFvWGKU+N138MfPTuH9438Xk3HCFHN0v8zOMQGPlri9KMaU/Qpfy6vCz3c8jhfU/zsonfF9lQjT7VlhSussy4/glZf07BclmKRCPnajZNF6OyQkH4UqYeZu8Yiy/8osQ4Yza4ausxLvegwkuzETB6LOCBIK41B2CTFNlEUO53jklZeckJUSS2nsluPcTmqZbylteIho3f1+V2CbkxV0CCznkhrUe7w1/FqyjARIgARIgARIIHUE7rvvPpUgU0Izz507p8I6ZLpQ8aCQdf61lcAoLJDQALyOb2gvhy1F+NHz5YbAezH1H7SRwfaE+FYd5v5gHoqNWpKbQgwS5uwao+evwBTsw+a92t3CSng5ZY6R08KU4bdc+7qaDWTK814vCr/KLMs4Ap+eQbXkkthajTPi6SC5JarPJC5PxNhCVx6KOXOK0cvwqEg1r4yblcMBJFN9Vm5FyaIfGgksJbmHFbLh1HMtlMAV4jHkUaxf/6hR4wx2yUvkJKMonRe1UeKOVtIyTgDfx1dH9NSFgb8y88bPL0/Ao3/2XUy+J7yaGCdeNIs/3oNdYhw2y9J22TYM6BktIulZUugObYlU17OtRt7US/KB/Hzki9dE6WJE7TKW/pzZOBajtGixu9f2zs6hPERK2jxmtzJcIwESIAESIAES6EgCMnOH/A0ZMkR5SMjUoU1NTbh8+bKawaMjdUuLvu2QDX9dwvNDuOtZhoepRqHMiiG5J8y/0fM34I35RkmtlYTxIaMofLE/hoq3xelL6um6duNK7C1fgTd85w4Nbw1J+PjSPsh0orFMe+ojoesUtesY6DhMzSf3I/fR2Zjd6wyqq7ZiPwpRNvO+8DANQ8VeuYWIxe2hV99cYP9ObLHzZxaWzUJZYS/YE/sYElO3mJmGCTFKfHMdsEg8HDyZPRQ7O2TD3KRCPIoxyyzzcrZzTBTnR6rkbdRB62FGCa1HjMaJkz/DGjXzxq98jRJamutX5ZiYgAneaBBXpfRZsXJI7LY9BMLn2IyWgyKukehpSSWUw/aaqFlXBDUpSFyCrMo1W8TAIbOJaC8NW4jdj8zOMZPTe7aBLJuQAAmQAAmQQOciIGEdMj2oTB8q/5L08u6771aDFCMF/4SAHbJhfl1T7v1FmGuWxQBL5ZgY8RcRP9QFJc4MFn8M78o0l1iJJ9WvUfPks3jyt/Pwo38wPCnUlKSSi2KVywvDaMXFMAKJOwbCRCepIPe+2bBmAh2CibNnY2IM/VgGh2Z8Wga3AePTZtSLYUMnn8gdizlzxgKf1mP/tv2o378N9fsLUTarDIW6Tgz9JbJKBoZynMGub65DzexKf6NE7gio9BFHHB97xevMka0yDYfjrnJm10IsDCWYsJgq48VseCb0SCTvhMmq/+BfcK7bMNzdw+8fOH32MKyJony6PPkzvHC8CmNG/gpf03klXNWOqRCONYfc2S+tGT0mY4yPd4WreZqs5M9cpPJDYOW74XlHGquwzpqyA4sS8IJf864ShhWLdAhIHDkuwnjVQIl7ZCZ0GIdTpWSSGlOiZ+dw5HOBBEiABEiABEgg4wjIrB2S6FKmB+3Tp49KgimJMMVI0eX/dB6I/fYUnzaQ2v37gBFFwQYGNQXlP+NtHWmh2lmJJosfGme3O4YNTy3ABjVFR4i0Ml6U/5kzpWftxgV4cqO7fz1TRPHQ/gAkbERmlTD/rRkmxCPiDdMogWPYINNilq+gUSKEPPJSW4+ByFLTc6tKcnkE9Z5Els31R4DCIRjgNTr0KkSZ5JiYMx1jPckxUz3ADLtaNaNmXSW2lizCDx/1n2kByEXJrNnA1m2o0TvkzC5UbgVmzypxDBNq+lBX8ssz2KVCQ8oQJDnVOydSf4Wl/wPPPfzj4P+/mADLVu6R8vEerIlolJD6o/Do4AloPLsG73xst/94D/63eFgMdeei8EhPs9USLLKyQGKWazaKGqyzQy5WbFsElS6inZrrGT5UaIfIqtmCxVFjOgI6rXnXmubUN+TDzvegwjkC2rOYBEiABEiABEigyxCQBJeS6DIvL08ZJ8QYcefOHWWkuHXrVpfhEDzQezF1Tjmwb2vIyGAnsXTlc6h9Xc2W8V2d80G9zLqTX1qJJssx10lOOQqTyj3JL5Xs4ZhvJKMc/fg8FO9b6TJgqNANmLKCRxDachFv/+NK7B0xDz+az2SXIS7RlmI8BqKJyYTtvQoxoazQnciy+Qh2HgHGji20vSg+Rf3+LdhiTsOhphktRK7XcJHCMWdUKIc1A4fQWYdvLvRQEmPFItvwILkjKoGFRqXZlevduSh0ncqF2GqLknwV/qEhnr4ydtWegUOmmDz+OF447h5I/uDXsWy8FaeRP/67WIbvYM1/PK7ySkjNYA8Lt5y0WitZhLq6SVhXNMudq8EIuUiEvvkzX8IrVbuxeHEpVEoIkX/gFTxfuhgrZ63DpKjJLrUWjaiyXDmwQlx/wv507ozdWLylBjOdBJVhFVlAAiRAAiRAAiTQBQicPXsWw4YNgxghzpw5o7wkZKrQnj17MseE3v+SfPL51/HktxZgo1025fkNUXIzSH6JVcA/PotvPPW63aocL7w2z/GEkELJL/ECFuDFp3QMxnDM/4EnGaVM8/kD4LvfWoAntU5iXHjNCM/Q5RF+LcOIVJCEnFonu4HIc3lWRBDU2TbZoS0n9Lgk/GUTAMnZoQ04bToGtMDM+u1VWIZZ2I9tW7bYikuIxhwjRKMXJKcE9m+DUwXA2OlmndSPOQurft0q3Z5cGB61cuHcORQVuv0Hbt++nXot2SMJkAAJkAAJkAAJkAAJdHICly5JIsT4/g4ePIiioiJcuXIF0l5m5BDDhIR0iDfFX/7lXwYK7N9fwgj4RwIkkOkEJJwrHf/q6s9gwKBBvqqNWF+tyq8tnax+MyyUw3dMLCQBEiABEiABEiABEiCBLkfg5s2bKtGlGCEkhEPWc3NzIUkv5UVFDBT8IwESIIFMIEDDRCbsJepIAiRAAiRAAiRAAiRAAh4CkvBy8ODBuOceKzO5GCgkv8SNGzeUYULCOfhHAiRAAplAgIaJTNhL1JEESIAESIAESIAESKDTE+jRI770b+IhIUaJ7OxsFBYWoqysTHlNjBgxAnV1dSrEIwhavH0FyWE5CZAACSSCAA0TiaBIGSRAAiRAAiRAAiRAAiTQTgJ33323MjJkZWVFlST5I65evapCOMRzQrwjpEzyTYjHhKzLr/dPZIshQ/riHwmQAAmkC4H4zLLpojX1IAESIAESIAESIAESIIFORkDyREiOiFj/Bg4cGFZ13LhxkP+HH344bBsLSIAESCBdCdBjIl33DPUiARIgARIgARIgARIgARIgARIggS5AgIaJLrCTOUQSIAESIAESIAESIAESIAESIAESSFcCDOVI1z1DvUiABEiABEiABEiABLoUgaPXzuL/NPwOvbr1QEVBGYb0tmbb6FIQOFgSIIEuSYAeE11yt3PQJEACJEACJEACJEAC6UbgFw3/iXM3ruDDa5fwf8/9Z7qpR31IgARIIGkEaJhIGloKJgESIAESIAESIAESIAESIAESIAESiEaAholohLidBEiABEiABEiABEiABFJA4KsFf4pBvfviC5/rj78e9Kcp6JFdkAAJkEB6EGCOifTYD9SCBEiABEiABEiABEigixMY87nB+KfiWV2cAodPAiTQFQk4hokR66sDxn86oJzFJEACJEACJEACJEACJEACJEACJEACJBBEIDZ7AkM5gvixnARIgARIgARIgARIgARIgARIgARIIOkEaJhIOmJ2QAIkQAIkQAIkQAIkQAIkQAIkQAIkEESAhokgMiwnARIgARIgARIgARIgARIgARIgARJIOgEaJpKOmB2QAAmQAAmQAAmQAAmQAAmQAAmQAAkEEaBhIogMy0mABEiABEiABEiABEiABEiABEiABJJOgIaJpCNmByRAAiRAAiRAAiRAAiRAAiRAAiRAAkEEaJgIIsNyEiABEiABEiABEiABEiABEiABEiCBpBOgYSLpiNkBCZAACZAACZAACZAACZAACZAACZBAEAEaJoLIsJwESIAESIAESIAESIAESIAESIAESCDpBGiYSDpidkACJEACJEACJEACJEACJEACJEACJBBEgIaJIDIsJwESIAESIAESIAESIAESIAESIAESSDoBGiaSjpgdkAAJkAAJkAAJkAAJkAAJkAAJkAAJBBGgYSKIDMtJgARIgARIgARIgARIgARIgARIgASSToCGiaQjZgckQAIkQAIkQAIkQAIkQAIkQAIkQAJBBGiYCCLDchIgARIgARIgARIgARIgARIgARIggaQToGEi6YjZAQmQAAmQAAmQAAmQAAmQAAmQAAmQQBABGiaCyLCcBEiABEiABEiABEiABEiABEiABEgg6QRomEg6YnZAAiRAAiRAAiRAAiRAAiRAAiRAAiQQRICGiSAyLCcBEiABEiABEiABEiABEiABEiABEkg6ARomko6YHZAACZAACZAACZAACZAACZAACZAACQQRoGEiiAzLSYAESIAESIAESIAESIAESIAESIAEkk6gR9J7iKeDGRPROi4HwC3se/sdTDlgNi7E3iUjUd4buFx3HP0215sbU7Ac6t+vs9rDezBmh9+WZJdF1guXGpC14WiylWi3/AeHDsTQ21ewrf5Gu2VRAAmQAAmQAAmQAAmQAAmQAAmQQOYQSC/DxI5qLMudjDVF2SifNAFLDxzEWs1yxueVUUJetFNvlNBKyK/XaDIGR5cXYPS4aTiKjjJO+Oll6pzey49O+RP84k/6Ik+x/R2mHKBxIr33GLUjARIgARIgARIgARIgARIggcQRSLtQjrWbG1Er4+vdD0/P0AMdg6Pak+L36fb1/yh+WndLKTp68BitcGJ+Syegafk0NM0tTIy8GKVsXDANrcsnYmOM9dtTLWSUECnZKJ/637C3tHd7RLItCZAACZAACZAACZAACZAACZBABhFIO8MEcBRjDrcohKOLJ2ApgKVz8zEaEsJxyhPekV6kL7dcTaxCA3oiL7ESY5BWiKESTZOCP7dRQndI44QmwV8SIAESIAESIAESIAESIAES6AoE0iuUQxPf8RH2FUs+CfGaGAMUZQNowZuuvBLhuRXcuSf0djP0QpcBTk4IO6+FtK3EMBVG4mzT+kT8HYOnffUDxPOgor/R2JvvQTwipvYzjA8hXc22eUUj0bp8ZDtza9ghJ2jBpsNAhfJAsXRzxuvSJwcVy6ehQuqvrsZ8X07HcU7tJ0/eD50r5EYTlr1shOMYKPyNErqCbZwAwzo0Ef6SAAmQAAmQAAmQAAmQAAmQQGclkIYeE4K6HlPebcJlAKPHFdjeEh9hvrMXtIFBXuT3IGv1HiyruwV5gW9z2MMAMUrcxKbV0fJEyEuzhDrof9HPfnl39NNGiZB+WeIF0r8ArQvscA9tBJCX99UyhgbUqlAGK4Ri/oY9UG2Up8hxNcbIuTW8eln6HXXCYbRyOagYfEXJs/oUxnbYxoGD6Lf6OPapFA8yJtGr2uAOwMUptJ/yBuQr7xbpZeNgy+Wi9kRbjBJaT3pOaBL8JQESIAESIAESIAESIAESIIHOTCBNDRMADhzEm5ds9DeaUGl6SziJMBud0A6dmyKv6PNtyo2Qh6tY5n0J993zhrFBvbjvwaZLlneBYxQpnYAnxFPixlW8qWcW2XHFyp3Rv2+Afkcxxs8Q4KuDX2G4XmKwCZ8ppAWbnFk6ruKcMkL0xNBSP5nhZWGcDjTikMjo3RPjVXUdCtKCap9ZSr5QOsFOdGnL/uw6TluRO6rg8pUWnHO6pXHCQcEFEiABEiABEiABEiABEiABEuikBNLXMAHgUIuVVBIt10Ozc0jOidyeane4czrE/5Lt2qeePlzboqzM/73l3eEYRXRuiN79sMblWSGCbCOA80Jv1pmMvTEaCKKolLzNYZzq8eYF2U85mCjeGaX5GC+5Ky9dcXta2BpNL+4TCl357DrefPug7aFhV7jyESa+3eQyTowe1jd546FkEiABEiABEiABEiABEiABEiCBDiWQnjkmOhRJGzo/cB3nXLkibBnenBIu0fWY8nK9U7J0rj1N6tTJ2It3HE8Qp0IaL6w9cRWVRf0gs5IszbUMD7Vn/WdP+Z87TuGJr43A9F438Isdh/DXtTew0WOM+fDAQUzEBFQL00vnsGz3+TQePVUjARIgARIgARIgARIgARIgARJoD4GMNEysbb6JNchGXk4fY+x9MEjNMnkTp3X4hLE1qYuld2GQ2cGFm7iMHOTl3KXyLqxV24zkkz4hI2s3v4PxKllmNgYNMIVlwLJ4f0zqh/L++ajMsRKV+oVxqJE012PGqyGDTNDoxDgxONX7MUgZlpMACZAACZAACZAACZAACZAACSSNQFqHcgSOWmbtkLwG/fOd0IfQlKI6SWa9nbvAeNHXuSkCBbdlQyH2TrJm1rhcZ/et82OoWUUsmV79xENCJdDUyTARnJvBbZZsB3YAACAASURBVIBpi45taRN73glJVmqFc2QjL0IYR1u0YBsSIAESIAESIAESIAESIAESIIHOTSAjPSbUrB0vA3uXjLRmyJhq7SRn2kt7n83f0ICJywswetw0tI6TvAcNVqJKcwrPuPevPfuF3aduHt73HkA8IHTf9uwaemYN8ZCACt8oQOvyAluMZ3aPHdXYNFimHLXquKdD1T3rX3+9ZMxZTrJLXTfSr8y0ka+mMVWzj0z16BTQVJKPPi2sAQSFcQQ0ZTEJkAAJkAAJkAAJkAAJkAAJkEAXJpCFVb9u7cLj59ATRiByqEqkblbMnYzvFUkICHDuj4cx+E3mlIjEi9tIgARIgARIgARIgARIgARIoDMQuLZ0shpGhnpMdIZd0LnG4A1ViWd0Kze/g5XxNGBdEiABEiABEiABEiABEiABEiCBTkOAholOsys7aCAzJqJ1XI7V+aUG6FCVDtKG3ZIACZAACZAACZAACZAACZAACWQYARomMmyHpZ26O6qRtSPttKJCJEACJEACJEACJEACJEACJEACGUIgM2flyBC4VJMESIAESIAESIAESIAESIAESIAESCAyARomIvPhVhIgARIgARIgARIgARIgARIgARIggSQSoGEiiXApmgRIgARIgARIgARIgARIgARIgARIIDIBGiYi8+FWEiABEiABEiABEiABEiABEiABEiCBJBKgYSKJcCmaBEiABEiABEiABEiABEiABEiABEggMgEaJiLz4VYSIAESIAESIAESIAESIAESIAESIIEkEqBhIolwKZoESIAESIAESIAESIAESIAESIAESCAyARomIvPhVhIgARIgARIgARIgARIgARIgARIggSQSoGEiiXApmgRIgARIgARIgARIgARIgARIgARIIDIBGiYi8+FWEiABEiABEiABEiABEiABEiABEiCBJBKgYSKJcCmaBEiABEiABEiABEiABEiABEiABEggMgEaJiLz4VYSIAESIAESIAESIAESIAESIAESIIEkEqBhIolwKZoESIAESIAESIAESIAESIAESIAESCAygR6RN3MrCZAACZAACZAACZAACZAACaQXgZzs7vj7sqEoL7oHhXf3SplyBX1S11fKBpUBHZ26ch27Tl3C8t+cwNWbtzNAY6oYLwEaJuIlxvokQAIkQAIkQAIkQAIkQAIdRqB04N145cujUJTbu8N0YMepJTCs7114evwQPDK0P/5q23s4dOFqahVgb0knwFCOpCNmByRAAiRAAiRAAiRAAiRAAokiMKWoH40SiYKZYXKG592F//7FARmmNdWNhQANE7FQYh0SIAESIAESIAESIAESIIG0IDCzOD8t9KASHUNgFvd/x4BPcq80TCQZMMWTAAmQAAmQAAmQAAmQAAkkhsBdPbphWF+GcCSGZmZKGdnvLmRlpurUOgIBGiYiwOEmEiABEiABEiABEiABEiCB9CFw/bM76JbF19L02SOp1+Su7O5oTX237DHJBGiYSDJgiicBEiABEiABEiABEiABEiABEiABEggmQMNEMBtuIQESIAESIAESIAESIAESIAESSDMC9JlJsx2SAHVomEgARIogARIgARIgARIgARIgARIgARJIDQGGcqSGcyp76ZHKztgXCZAACZAACZAACZAACZAACXQ4gcbP8NbxO6i+DNyTl4WHx3bHyDz5ZtuKTz9pRa+7+f22w/cRFehSBGiY6FK7m4MlARIgARIgARIgARIggc5K4Dbe2nMH48qzMbh70BhbcbbmFr59Cjiiq5xvxRvnP8Pce7OAT1rRc0gPLHpQb+QvCZBAKgjQMJEKyuyDBEiABEiABEiABEiABEgguQQa7+A3V1rx6lu38KOHsjE0x6e7y5/hddMoYVTZfLEVE/tmYe6t2zj8cTeMu8fYGOPiey/PxLM7IlcevfhnWPPYwMiV0nbr+/jxYy9gO6Zj1fZn8EDC9DyPf/vG1/HyMSCz+SQMSJcTRMNEl9vlHDAJkAAJkAAJkAAJkAAJdF4CZ6+14qtv3cQ/PdAdDw93u05cOtGKf40w9OorrXi4uAfK2mCUELEPLKnCziV2B+d3YNn8n6A24S/xEQaQJpvOb/87PPXK8TiMDANROBzAsTQZANVIOYH0MkzMmIjWcTm4XHcc/TbXh8HYuGAaKvrfwr6338GUA2Gb07ygEHuXjER5SwOyNhxNc12pXkQCpRPQNLUfzh3egzERLeL2Pu9tS7vRhGUvH8TaiMI9bdCCTaurMT+sTQz1bD3zvG0vRTgGdRvfOjH06e2rvevqmtAz4ee8upbkxLI/2jsAtieBCAT0+QYE3vcitOYmEkgsgTiOx6VzJ2NNUbbdfyzPZbHeP2KtFxp6bM+GWq7/PdWSoWXGMh5d1/NrMNRbgp5p9faM+o3lnmwziP6MlKSR53dDWfZt/PoWgDvA3x+6jVeP3MaTRd3wpeJu6J/TTZVH6v3Jsdn4yhc6+ZwPtsEE0Tw33nsV05/bice+V4W/TZxrRCT8atvwIZnqTRJ1aKwQgUB6ZXXZUY1ldbeQVzQMe0s9WpdOwBMZa5TwjCUpq3LTnYbWBWMSI11uLMun4eiMxIjrelLshyAxRK3eo/43tfTDmuUTsTEQxhgcXT4S4y8cd9osq+uJirA2MdYb0BN5yrBh9a/1CDaMFWLvpH4IM2QofX3GcynHR7fAwSVvQ6qO1VT1EzepBJ/7cffPBm0lsPHBfsgTI+DqPb7G+LbK7czt5AWydckELO3Mg+ygscV6PFpGiZvYZN/bsg7fRPnUSM8LMd6zEGs9A9CMiajob6wHLC6dOwzl+iOBp45jqLbHs6wOKJ86Ofw51NPOuypcWqf2waG3zXtuA84VjUzPY1aMDMvjH6d33Gm3frsbvjwGmGAodvYWsOrkHXxl12eY+MubeLbB2OizeDc6uVFCxnz+FGp9xu4tOn/mA29RzOsDH/s+dm6vyuCQlZiHyooJIpBehgkAazefwr4b2Sh/0HzBtl+YLjVmoKdEgvZUVDH1mPLynsR5Yxw4iH6ro3kERFWq61aY8XmU927BJsM7Zv6GBtQiB0/MLfTlsnRuPkbfaEKl4S1knQ/uNrHWU53cuIlDvr35FIrOaMK+SwHbvOP5fRMuIwcTO9p4lapjNVX9+OCPXJTgcz9yZ9yqDFSRDIyxIipUsc+XW67G2oD1AMzfsAdZUT3PiCp+ArEej2PwdFE2ag8bnnw7qrHpEjC62N9gFOs9K9Z6obEVYm9xDmrr5F4U6U90BvbVtYRX0h+93g15M67d3IhaZGN8sf+9OlwIgBkTsUb6CPPoPYoxbzfhcu9+eDqF90oxtjQFPGs4+u+oRtbqTPRAdkbgv/DxZ1h1GOijHXp8ah2541NoFPWM0Nao1q5FyUMx/bGZWLb9vCPHr0zCIYLqSbn6f/l9R4Ze0LL86qhtz+1UVWtf+XqYfEuG5HuYqUIxZH37c1ZfP35P92D/iueF1uOxmTC3++kO8cAw6k9/7FV4RXp64GoXIpB2hgmgHlNOtAD980PWap+XvC60jzjUDCSwcXAOcOmKJwTjKKovAXkD8n2/9o3PyQZarntCPepxusXdJtZ6S3N7xkFuDI6Oy0HtiYM47dNKyfIaOQ5cxzkAg3LjeHjzkc0iEsgoAsoTKaM0prIkkDgCM/piNFpQ7QljnH+2BejdB094vV0BxHrPirWeHozygkATfnpCl/j/blxQgNHyYavZZ7s6n2/itCs8+CrO3QDycvr4NPArsgwkCPp4Zhu1f3ohVfdKy8jkp2mXKMvvhsm9gXcklKONfyML2tgwjmYPzPkbjAZQe/KC3eo86m3nhNq3DsIyV5zHf711HMB0LLATZYpR4dkdI7FkY5XyRtj5venAjhcw3TBORKujcmBIO1hJJv29GgbiKz+qwmuLR6p6Esoh9dzhHDvx7PeA57fLthfxmDJgRDA02GEhmPGipftGYbATz35jhz1eYOAI6W8kRjCSwz4uutZPeuWY0OzF+j54GiomTcDSA9fxtJ13whVnL1+tphpu596YeBUHB3d8vrdMyRC3u0YMmlqgvlYH5gBQbY3UvmZ/jhzD8uxXZo/PFZvpzTugdTwMVIwL9Vcr+QwuuMesypyHA8PVXn+l9zLy7SvUB8LG5M2jYPfhuEN6YjX1mA9fxfhxoX0TPb7SKzc8vtMd/+kTj+3ZP94+VXs0YFlLvoqLNdm5ZYf3DRfHW9h3ONoXTvvL04XweodabgH9e2K8Pta9vzl3KaNFKA9FwANGrPVagCeWTMMavc/MfWz0bT24NSBrB7BxgbFBL544hSzDk0MVl96FQQDONYfng9HNnF+bIVz5Y8RtVx4YzZwX1rEg4SzuPDN2XVuguf/0/nHHs3qPKcDVRivmOm48x7Ouo3/tMbj6cbUHXOeQbmf+uo4l2eDpM9L575zrpkBZtsdq5q+JJCeG64jwfzNnZMhF2nPtaNv55KOnqK90NfKIeBn5Hh8NODSgIOSabes3XuUhsvl4dFalnv3lvk7oY89ftpyT5rWiYvk0VHj3n9219eM9Bo19bY5R3LyXj/TPMWHXc583PhwjjsvWG+Z55lOmj5m3b+IJube6uBsDUzoZ9017k5tlSH74NTcCF++xoGS7x6v2gTc/TCLG79m/0rXvNcNB4R2Hz73Doxe8x6R5HIhcL/MI+8Q8FgGfvh099YJX3zYcjwAsI/XVcE+8CzeVB92gAbo/z2+s96xY65VOQKXyUBBPhwmo9HTnrKpQDxnrUXWtccrthaUXPkLWau89rA8G9QYu+9zDve2t9djqrz1g9OM9j8xjI9px4d1uXofMbZGuLaK43/lmtpc63mPSARDhniwfLab2c2qmfqE7vjT+Dr78bit+3ZbOP5eFL+pnpra0j7XNwCGQPI+1O36L95bcjwfOH8RbMhvFqJGoPXZKvagPxAWclESQMx6yZr44vwMb5Flg1DT8iX5xf+AhPIad2O7IiaFOrDpGrTcdq340A5YqAzBilCSu/AD154EHtH6RZAycgTXbU+hKFEkXbksLAmnoMWFxmS9u4r37oXJJuHu7+SJixc0fx76cgjbmV8hG+dS+qJbYwiD3ULlQy9fkwzpusAG1/Qvaln+hfwEqccrOIXAc+9APa8LiZXNQMfiKkWfgFkaPm4bWSUClEwMpZZHcicfgqNwY6nS+AquvSu3aF/eYjIdDRwe//AfZKC8O6Zl1uMU/Z4hx+G9cMFKFECxz5ALlYpSy61gPYPJgYfN/uwkoGhni79xcQ/tHYjrD3BgN9jpppfOQq2WrWFkj5tK+ScvLqHWsnQKKQ0YXYxjGovWQYhTEtOgc83ofyUOgjovtHTJmxFpPfX3q3wd4V+tuHbdheUj0g5s2aPlo63qgUtvt8KobTfhp4AuzIejAQbwp3iLmFyj15Q2A/SCqapfmY3zvWzh0wniAg3GOrt6j8tBEPvbtYxVN0MeUHIejxxn7VTrr3Q9rnPPsOPbdyEFFPDla4j6HfM5J3z79zv9I57rB2bXoJye260he0UhMPKuPm4DrVNznk/aG6+vKs2J5F9lhemHnm/+1Pa8o331cy75cPs3QuQG1cv8wzqXQQ7gelxX77b1O+MnWclQYwWFxB9fXI8Od3cU+yvVSfUGVY84ysgbmmPA7b+xzZN/v7STKsV7/XPoFreSgQt9nIlwP4Dkns+xrspclwo6RKFx2fKRCOV0u9N7xelVP0Pjlg0FFf71f98C6ZgSfd9HuW9bxZj432OeRvsbEeKwD4fsk6n3Ly8g0Xjr3WeP+HevxqL0fwuRHLoj1nhVrPWWMnSTPN6eihPfanoBm2IlH1fB7mxjVCpRXyJteY7ynrbMaj5HeaSQLxr1NP39GPS6i3Efi2JcuVWQlat+6haG3HE/q/urNMXIL57QjgG6Wyt9BPfD3k7rh2T7AF+Ls95lh3XF3nG3aVv1+lKt3cutF/vyBPWrmjgXzxVyxE/skvuG932I7gMe+dL/Vhc4LcewneMoJh5BpO+XPkuPkjohUp20Kt7+VMqLA8vBw9I/gYdH+HikhwwikrWECBw6iUhJh9s5W7uWhL8iAStDkeiHSD7xG+EccO8IVKxnYznRdPIoxq/egTW55lxqML8H1mPKuGGC87o+34Dx0St6N5ptQX0TMGEhVFqisvcF8wbPi0CtdLo9xjEmF03j0UnGY7vwH0rGEAzj7a8cVFasZ+BXF1vTyhUanzdrN7yDr3UZ7yxhM7C9frowXANst0jIu+LlQHsVPVRLVz7tegORrlZm/ARDZt7DP4ArPg7HvsSb7zNYuoT8yLvsBv3X5NMi/GLEkISzMMIoY66mXKFf86FGMkZcqM0xKHliLZSacjzxhJ5FHph7Ke3vYRW4C5erbP/RSKi+kl+uaUGsc/0uL+yDvxlW86XKrla+Bof2/9sRVXEZPDPVxGVYq6BcZ1371iaWVL1TOy1c93rxwy20kiTIea3Mc5xCs60bIEySoT895Fm28gXp65MRzHbnUYMw4E3Cd8p5PKlbbe/5bL5pOziB1PTDzktjn91nrJVvtf2/fKrQvdNyo4bpcpi23a/mypw2OgF1msLGON/NFxr5OeEOrXLLt8CvToGbIDFyM43oZKMPeoF7WjPPGfY7Ecf2L1pHaHvs57b0mq/u1l2XYMSKGR3fuHSueXx8T1jlhhru5x+sdRILHb4bfqfj7j3Ao6DojM6kE3rcAy+BmHpPuXDC+9xZvGKvvPol+3/JSQgKPxzDZsRTEeM+S576Y7oFqPC2IZjjQOStiMp7b47AMVJ5njljGGFZHPAqs+7i+n/slbXWdR/IxQu6BrmuZfr7V18BY7yNhCkUtiOuYNO7J0DlGBpu54aJ2l+QKWbhadwerrgIfxtHTPb2y8JVRqXs1euBLEk5xHG8deN8K2RDPCPvlfftv3sd7v5E8ED5hDToUQoVQ2CEd27+Pr5heCrHUiYNNYqrej781dF6lDDM78ayRZ8JKmOkZS2I6p5QMIJCeoRw2OHkhXyOu4i6rq+3a7o3Fj+ZG2J6dYbulWa67gOOOabrlxSjfP9FZNqK9uPuLt1/OXC9xuqY8mBeoTNmtU6VMvgRVY77WOc4xWfkK/OMwR8f70K5VtH8lvKHCdjmUIst9tt4yVKiv6rewz3UMmAL8XSitYycSH3GNk1jZbIyeOg0Wo5Bcy/AQcKxFdVO0XookzCHuP3kw8+xPMQKE/cVaz9vQc57oGN1lsX4ZUl4c8mUxFtdhT+fyUjquwEqWucN6uD70+4M4nTMNT0iSsQPAEwOyXQ/7HgkxraoHO3iP1ZiaxlcpznPIEu52f1VlN+LrNvG1w8+TNl2nVKz2LRxynauSI2WkY/BZC3nRL0CFeoAV12orXn2T8ropVPsfvQvQutwb4OuTtC4GECEPHcsAktffCptwNU0C/4ReLw804tCkkfZ5YzEKvRC34/rnghDvSsDXUMOzy0+idW5mw7mXGpV0dngxPFYWWcb6tQe84zUaqMXEjV/dM4pCx54OTVnruR5rDSLet5TRW7xhwsP5rPYB9xbP9Vn35fqNet9y1VYriTweVTiiEQEa3ltASaz3rKj1tBfEnsjGdFeoR4BO3mKVwFISe8aZ+Nv3mUAMCKGp4S0vF2+H3vVYr4HJuI/Ec0wG3F/DwnC840vtev/S7nj1k9v49hXg4xi7/kZpD8QwyUuM0mKopsMwTv4WwyWM42GJhRqoPCm273gNGyQ0wgzbGDjMykvxwRmcx/12CMX7+PFj4jUxHau2P4MHYqkTg2qpqPLAkp9hyQdfx8vxhH+kQjH20WEE0tow0WFUwjrWNxjbDdV2G650xcuHNYqjIOAhLw4J4VXl64zlDm+FQsjUjtNQ4cQLJntM4RoFlYiHhPKwUC65OVbYyjjbkBLUKGHlkV6w25qoynoZK/cx2Kjwihs+8bmB4wl4WAirH2s9s6F+CLLc4MUIGPqTB/T88AzjxoPblIAH9pAM75L1UvqEJMuUh+sbV/HTA8DaAS2oKJaEoNcxKCyMwysjndbjPIdsN1kVXmUbglS+maCY7HQaqtIlMdcp8ZypGGd//VNJYhvcLxjONSrxAPTLZuIlJ1OieBEMwxox5pTe5RPqlMy+45RtenYFNo1ybVeGmH72jAh+oV2Bgtu3QTwkxEBmn6cSztS6fFj4NdDuJX3vW+3DEK21ZcAJhRY69X0Nk85Wn4VY71meesowA0DCW8e5xcqUpa0PWrlUtBFMlamPM6G68iz0hPf5Tfa7nc8s5HkVahN5yfoYUa4NrpErR98a6RqYzvcR54OhvjdGH2pSa3TvjgnTumFzw21UH72D9Vcie088M7YHHh6U6mlCrXCO7Tt2YjtGYkmp5fKgPCl27EStGCsWT7ANEGKzmIEFM36CZ3f8BJvfm6ESUZ7f/poK5Ri9+K+sPBSx1DHAW8k3TVcLY6Ox+MGZWBNHGI08i1ZSTivh5hqVzNPOoeEYX2QmEDFUmHU8QrjaqQmkzl8pYRitWQpcceki274pur0r3J1aXw3cZfGt2a6Yq/eoqbFCX+N8pCh9wsvD2qh6Adbn8OZtKrFc+q3Y/DBuMguKTDMaZUzyMIIw93n7a1WiprtTrrN7IPHKjqu++oIUyaPEeiDwcvX/QuTBF1V2wLFmx5N6pLlWrSSX2vVSb7K+2oa+dupy61deUluXe2OaLca1tqu71IytnuVGetSbU0gdbzr8ILTvrfwZoeNAJbxyhYHoRFlWzHT8D27WGOWlVNy090oYh3aDFk8KCeeYaxkr3GEcbkaxrPkfq7G0bGudEEeZMs97LGqp6kHZ49quDFW6Qhr9ho0hluuU7/nkeamQMapwKXHdt13Sda4EBJxvCeHif51IiGgfIf7HYNuvlyp8qX9f7A0LdfIfV/Trn71ffHSPrSj8muw/o4Jbmj8Xdx09M5dcJzYqt/ZI04QnYfzyxV7lYZDpncPH6dVW3NjV9dO8b9mhRGHnkdM44FhX51kUA6DveeYI9l3w597G41H1r0NvQt2p0JWwMDxre2z3rBjvbZq3nSsjxF4+MoSmTFchoWYdWTbyw4RC6kLGKHeS2dDYoi/ZYXmuMEmjlR3mZpQELAYcF0bt5N1HAvr2PSa9oZTW9cTf085QviMWb9zBhx/ewb9GMUose6A75qUwhMNEYc1C4e8ZIWEcD9vGCt1GZtSQEAg9fedTrxzH6MU/g/Wib9WKpQ4eeEbJUTN6eKYt1X3J78DHlmPJKEBPK2pOB2rWi2VZ66VlTRdPj1F/g9ecBJqxSGGdzkwgAw0TgE6QFJoT2ifONOzmac293ZadGX5TtV8w9Qu5uPG55ry29fHrzJU0U+vtnVbSr2GcZeJ9sNxM9md/Hbct2lHH5O3Ozr3gxIqrF+T8+BJEeWWqdesF2kyaZn3psI01dvI31/zo8sVguZ6jW8dfmvlFrH0dNW+Clm0mEXXJhpUXwTX/uJ300XcsocK1m0+FJVPc6E2oZfelk1HqWGszAaNq48qnAsRWz4qNdyeJtFxgw6cxDekduCS62glgg40S1r70i6N15Mp52bsfyvubs3nIy0U2yosMY4XToA0LPseqSkS33Jucqw2yjSbxnkPq5cDIpyE6VaTUZ9RQPtpiW65T+nwqDiWu9YttlxdPyecxepzMhOTOJ6Kv7TrZpKgpHl/hBrtoA/Bu19cJM2mxeMBNQ8Tj1StG1tW9xftg7qnocwyqeHdEj4v3SLJWxYvgRg7Ki7yhTnpcka9/YYZSFaPv21PMha5ri84vYhhQfQVpLkZyY99zUxkr+6GiKBumUTZcZqLGbx8LOjGldKS+zAcZCaLct/ymPVcJKEPHmz7WIz7HhA8YkodBEgl7+YfuiT6NNPcHQzkA2nw82vm/XP3b1zInv1RS720+42tXUSiZpMtYYcr0jMfcpJfFELLpkiSF9HxckLaSiPxGbHNX6uMi6BoY233EMjIEG8a01u5f3Xf0YzLblaBcX+edpNXq+VM/o7n7SP7aHZytuYWV/3YTz7x5ExO338bCs8DBgI4n9MnCLx7OxleLuwfUSECxmnlC8kA8Y3k0eERaORWqsNN8OXfa+OdaUNN9GrkaTKOEFh9vHT8Zlixr2lCZKjQ0XajOFWGOSdcL6azHZsr26uUaN7SMKpehRY+Jv52fQGaGcsgXDcjF3nDl87q+yc2zeDLWOO5+Ms1jEwaN6xn3XpUbDuZOVlnfK3Rr6c+Jy5ekgn3ROi4Uv1x7uAG14/J1bef3cl0DzhXLg7Zd5NXbqdnOBfmqgIloNfMnGMn+oo/J2798FQb2LpEx6vjvKC65XhG+6+LyBxxdHmJnTX0WSnYoHh9YME3x1+EGrhhQn7HG6rKtZZsxz76yzePo7QbUTtUMfAcF9dXPy0v4rzYSg4Y11SxCcc5OPhNX3djq+Y1NuISOW5fQCCshY0yewyFU3WFte5I4D6ehKqEl2027vLf22pBNlqu6vHTFNPVoSFrAkt+xan1Niz/8JKALSUob9brgaaunQdbnpDoeruDp5QVq9oi1zvXE064jVi81oHpw/NcpfczJDBnWueofKmXlEegHaK8ZPUbn2m5cD6KeN7pxlF99nTCP4bZcf+XF8MFpqFD70X98vue/zvMTRU3/zfocgWfGGvFACb/WO+ekLUwMpU8sGRnK7XCjCZvqeqKiLbkCbJm1h69goiT3c9ZjicsPnZvuY2SPZ3YFK+xrdH/zOuFPJjHjD+kVur9Z+Y78rxn6Gmwcp2rKztB9y08v19SLzrEe4TkmYMj6PAu8b4W18xtf2+/f4de+aNdXzSsx97aw4bWjwPpgADXTl0zb6/qL8/og+2W+/VLuPCtKktS646jEMKyJJWzPOS6MY8u8BsZ4HxEjwxNTbd6xjsPpO9ox2YJNJ3oaz2RB10EXzRStdMPgkm5YMfY2Th+/g+qGVrxzDTh4J9T9cRTJHwAABJxJREFUhF7A5PxumDiyG4bmZeT32dBguEQCnYxAFlb9Wj9XdLKhcTgkQAIpJaC+DPXEm5JkNaUds7PEETCmc3RmLEmcdEpqOwHx0Fkz4CqW6WkF2y6qfS3tL8AyhXKw91T7umBrEiCBFBBQeb16BuZRSYEG7erio7/5i3a1b0/jgj692tOcbRNEQEK5+Nc5CFxbOlkNhKbCzrE/OQoS6HACKgTHnG6vwzWiAiTQWQjY4WleD5POMjyOgwRIoIMIJDfHWQcNit2SAAlkKIHMDOXIUNhUmwQ6MwFx71Wzq3TmQXJsJJBSAnomKECFAaRTuE9KObAzEiCBxBGQHCkFatpJCTOhh2PiyFISCZBA+wgwlKN9/NiaBEiABEiABEiABEiABEgghQQYypFC2GnaFUM50nTHtEEthnK0ARqbkAAJkAAJkAAJkAAJkAAJkAAJkAAJJJYAc0wklielkQAJkAAJkAAJkAAJkAAJkAAJkAAJxEGAhok4YLEqCZAACZAACZAACZAACZAACZAACZBAYgnQMJFYnpRGAiRAAiRAAiRAAiRAAiRAAiRAAiQQBwEaJuKAxaokQAIkQAIkQAIkQAIkQAIdS+Dk5esdqwB771ACxz6+1qH9s/PkEKBhIjlcKZUESIAESIAESIAESIAESCAJBKpONCZBKkVmCoHNteczRVXqGQcBGibigMWqJEACJEACJEACJEACJEACHUvgf73fgFNX6DXRsXuhY3o/0XQNPz5Y3zGds9ekEqBhIql4KZwESIAESIAESIAESIAESCCRBJpufIZHtx7CG0fOou6TG4kUTVlpSkDCd9b8Vx1KXv8dLl2/laZaUq32EOjRnsZsSwIkQAIkQAIkQAIkQAIkQAKpJtBy6zaee+dkqrtlfyRAAkkiQI+JJIGlWBIgARIgARIgARIgARIgARIgARIggegEaJiIzog1SIAESIAESIAESIAESIAESIAESIAEkkSAhokkgaVYEiABEiABEiABEiABEiABEiABEiCB6ARomIjOiDVIgARIgARIgARIgARIgARIgARIgASSRICGiSSBpVgSIAESIAESIAESIAESIAESIAESIIHoBGiYiM6INUiABEiABEiABEiABEiABEiABEiABJJEgIaJJIGlWBIgARIgARIgARIgARIgARIgARIggegEaJiIzog1SIAESIAESIAESIAESIAESIAESIAEkkSAhokkgaVYEiABEiABEiABEiABEiABEiABEiCB6ARomIjOiDVIgARIgARIgARIgARIgARIgARIgASSRICGiSSBpVgSIAESIAESIAESIAESIAESIAESIIHoBGiYiM6INUiABEiABEiABEiABEiABEiABEiABJJEgIaJJIGlWBIgARIgARIgARIgARIgARIgARIggegEaJiIzog1SIAESIAESIAESIAESIAESIAESIAEkkSAhokkgaVYEiABEiABEiABEiABEiABEiABEiCB6ARomIjOiDVIgARIgARIgARIgARIgARIgARIgASSRICGiSSBpVgSIAESIAESIAESIAESIAESIAESIIHoBP4/Y+5MUbxmo1MAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_outs = [0.45,0.44,0.45]\n",
    "results = []\n",
    "for idx,drop_out in enumerate(drop_outs):\n",
    "    #print(path + '_'+ str(drop_out)+'_ensemble_' + str(idx)+'.h')\n",
    "    model = load_model(path + '_'+ str(drop_out)+'_ensemble_' + str(idx)+'.h')\n",
    "    predictions = model.predict(test_x)\n",
    "    results.append(predictions)\n",
    "results = np.array(results)\n",
    "results = np.mean(results,axis = 0)\n",
    "result = np.argmax(results,axis = 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.reshape((test_x.shape[0],1))\n",
    "ids = (np.arange(test_x.shape[0])+1).reshape((test_x.shape[0],1))\n",
    "result = np.hstack((ids,result))\n",
    "m = to_kaggle_csv(result,['ImageId','Label'], '4-sub_adam_cnn_2c_1p_2c_1p_valid9958_ensemble.csv')"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAABCQAAACRCAYAAAAb3mncAAAgAElEQVR4Ae2dC3QW13Xv/wIEOAIhmYiHsGQeCkYGYyK6GlBrwsMuD5fg3q4Ludd1ATXtirnYELfUj8YNTtLYMW0CtUpyVxsB19d3Ge5ajWVf80jMI84KkK6iEBssmYDB+nhIyIpA4SkZdNc+M2e+M/PN99Tje/21FnwzZ85jn9+ZmTOzZ+99cvDyO13gHwmQAAmQAAmQAAmQAAmQAAmQAAmQAAn0IYEBuq1TX5mhN53fi01NKC25y9nnBgmQAAmQAAmQAAmQAAmQAAmQAAmQAAlEI9AYOIsRo0b5Zpvwb4dVej/fo0wkARIgARIgARIgARIgARIgARIgARIggV4kQIVEL8Jl1SRAAiRAAiRAAiRAAiRAAiRAAiRAAv4EqJDw58JUEiABEiABEiABEiABEiABEiABEiCBXiTgxJDoxTZYNQmQAAmQAAmQAAmQAAmQQBQCt27dwu9+9zt0dnaiq6t34s7n5OQgNzcXQ4cORf/+/aNIxMMkQAIk0LsEaCHRu3xZOwmQAAmQAAmQAAmQAAnERKC9vR0dHR29powQIUTRIW1IW/wjARIggWQToEIi2SPA9kmABEiABEiABEiABEgAwKefftpnHPqyrT7rFBsiARJIOwJUSKTdkFFgEiABEiABEiABEiCBTCTQW24afqz6si2/9plGAiRAAkKAMSR4HpAACZAACZAACZAACZBAGhEQZYJYOMg/iTdx+/ZtDBw4EP369XN+06g7FJUESCCLCVAhkcWDz66TAAmQAAmQAAmQAAmkFwFRRrS1tUGCU0oQzGHDhqG1tRXNzc0qNoQoJ+69914VuDK9ekZpSYAEspEAXTaycdTZZxIgARIgARIgARIggbQkoFfhuPvuuzFu3Dhcv34d165dw5133omSkhJlNfGb3/wmLftGoUmABLKPAC0ksm/M2WMSIAESIAESIAESIIE0IyCWES0tLTh//rxSQuiglO+//z6uXLmC++67T1lLDBo0CIcOHcI999zDZT3TbIwpLglkIwFaSGTjqLPPJEACJEACJEACJEACaUFAFBHimtHU1KR+Z8+ejVGjRuHChQsQa4nPfe5zKC8vx/jx4zF8+HAVU0JiSZw5c6ZXlw9NC3gUkgRIIOUJ0EIi5YeIApIACZAACZAACZAACWQrAW0RIUoG+dfR0YGzZ88qZYS4bIwcOVIpHiSmhAS2HD16NG7evImTJ09izJgxGDx4cLaiY79JgATSgEBaKyQCu6tQO2IjVlfkh0XdXleNtReXoGZBiZGnHXXVa1FdZySZm0tf8OQ3D6b5dus7+N4vP8KCRX+Fe82uqPTvo0WnFfwTXqycpPeCv+HKB3Ok4FYLaqumYdUeU7Rn8da51agwk1Jy25B9/mYcrVmCopSUk0KRAAmQAAmQAAn0NIHDhw9j+vTpyiLi+PHjKC4uVu4ZlZWV+Pd//3eIe4YEsZTAluLC0b9/f7V/xx13qCCXEuxSlBL8A1C/DY9+Z7+DYu5zW/EX5c5u2I36mhX4tlNsPKq+9/eY53kYa9n3Tazd8pFTh3/dLdi7fh1qTjnZ4J/PPm7LW7ZyA16Y624wtvaC7WTTlozX62NDmZkMFL8zS/Falc+7jpkx5bdvInCoFu2Tl2Ky53X4ZuAQag8F7B6UYOaSmSgZlJodSl+FRHsdancAWB0JbAAHReuwdIknUz4qVtegxpOK9jpUr61G2WRTeeHNlN77H3woSgcPD1sZUXTP23hqgvSvAa/u/Gs8ezBUKeFbPpWR1FVjzOIXAZgKiDpUj1mMxWNexLNvncPqVNZKtBxErVak7KnFwZYlWOKek7pFv6W2CtNW7cH8zUdR05MVd0sqFiYBEiABEiABEvjkk08wYsQIFTdCYkKIe4bEixBLCDkmrhwnTpxAYWGhsowQSwhRUMifKCTEmuLjjz+mQkKAqJf7j1H1va2WMkHtrwAiKiVsBQKWY+Orc6wPQlLuqW8ChlLCUljMwddf/Xso/UbLfnzjqRX4katuu67SZ/DaevslWMnwTYw16gqe9Q34kaE8CaYDljLi7mB7MfXFrCGDt1v243VRHq2M1McGvCnKozmR8qTJsfaTEJ3D5MlueS1lRD4WLF0KpadoP44dtTvQviBUceEumZy99IshIUqDqipUra1GOAMHIIDdkqfqGxCdRax/gYPVqKtYjcoM1Ed8cPBhPLvzYbx6KZSGUjIU/BMeU8oIOT4JC0ZXAJfexQd29kjlQ2tMlZQ6VCtlxHxsPmpaQ1Rg9dHNmA/gxcWRzqPk96PlYC32YD6efVak3YPag44NS/KFowQkQAIkQAIkQAK9RkBWzRBlg/z+wR/8AR588EGlcPjJT34CWUVDlA2/+tWv0NDQgHPnzik3jcuXLyt5RHkhf/q314RMi4pbsHf7fmDO0qBlQ/lyfH0OsG/7/qB1sLcv9TtRc2o8qv6HrYyQ46rcR6h5q8HKbb8Az31uuaWMkNSiOVi9cry7blXXHHzd/CJfvhyvvfo4pnrbhSgddmDfnOWocp7NdSbrZbps5aJge7H0RRfP1F9RAj22Ao8+tQ0nw/axAT+SPI+9hH1h86TJAVEw7NiBHbuP+wjcjpOHApi8YLKljJAc+ZOxYDJw/HgAN31KJDsp/RQS+RVYXVODmhpx1QiHrwQLVJ4avLA0XB5vegDHd4gxRUVw8LxZ0nj/3sq38eKit/GUKBpcf224eA0oumOkK7UoX7QyAVxstZLDl3cVS6mdltpqiG0Enl0dalVQtASrn5WDL6K6NlVf8ltwUMwj5i/BsmVLlAJlT+3B8BNnAvSLltSohxhaRyQAj0VIgARIgARIoBcJiIWDKCTE8kFcMcT6YdasWXjmmWfwxS9+Ua2iIatrfPDBB0pB8Ytf/AJvvvkm9u7dq+JM6HgTvShimlTdijOngLkz3eb55TPnAKca8UmYXrRc+BjA3Sj2WKZ+dux4YP8vUS/lPmnESYzH2M+6KykafTdw6ud4z37ErD8kCpEvBJUITvYiFHnqR8t+VG8BqhZPcXI5Gy0X0IjxmHWft5CTIzs3iubghVe34rVXN/gocTSSSfgLlWerUkbp1LT8zZ+MpUuXYqm4YXg70B7AcUxGiceFI79kMhA4i4spqJFIP4WEF3oP7bfX1WIHliKDvTXCkpr8+bfx1OcLXcdb2sXnqAQjhruS02onELB8HeaXhFyqqh8lJWJ1AOwJaP8qidcwBmPGVKG2rhZVY2Tb+mfFG9HHrbQqryJD3EOMMqoeU9dhH5dy4iohecPGMRHBbHeN+UsqUVRUiSXKSELcNpTYzn+6rqraOlt+W267cn1cyVZV61Jo6GPBvog7i5SvRp2nPxFldaThBgmQAAmQAAmQQG8SEIuJBx54AIsXL8af//mfIz8/X8WL+MxnPoO8vDxcu3ZNKSmam5tx9erV3hQlPeq2X+K9SgN8thRl+BjnPc9VulNKqeBz/JMzwVgRVh0f4YxHq2EpM3RNLTjfCJSNHQ5x73hUfaWX329ib0jbLdj7L9uAlY8HrTl0NfKrXry9MSys+lE6mnHGTFZZun3zcjtEGxESLmJQPkoQQLuhkGg/vsOytBBrix07cLw9OdCokFDc23HyYB0qVleGapmSMy6xtXotgP/3zqP45183x5bfN1chirxKh9Z38OqFOhSN/q/uwJe+5VM1sQWB8D49bqHrAq6XdHGNWFUNfPfcOZw79xbEkOLFxeIC9LQk4ty5o9g8H9iz6mk4OomWWlSJe4gEnlTlJM8erJrm4xJS+zSmrarAW+cix6/Q7hpLKkULXoRKSyMR1m1jjyW0snh4yxIaVVVVeBrfVWlHLaHxtCO0G4N770UsPlipygUZ+PTFXYh7JEACJEACJEACfUBA4kgMGDBAWVCUlZXh/vvvV6trSABMvQyoWFdIoMus/1NWDOEohCoTnJzlX8BcfISafzHcOuq3GQEu5fFsCmZNAPZ9Z5tlMSGFlYWDobSAZaFxcss6FWzxNecr/UeoecooJ2XFtQPLsdoTxNKRyWejZd8PLNeSxW4LEJ+sTMoCAjfVR+XoHZU4E7uPT1ZxJpS1xYLJOL77OJKhk6BCQsYrcBDVdRWoLPPYtkQfy+TlEGXEwSfwi45LuHBudTeVEmY3GvDqL7+PloFfw2MeqwkzV2Zvz8fm7+rVLCpQqVw79qhAqFbcxyKUKM+XPXCMK1BpKTCcVTB0nhdx0KMY2YMlOBp1hY+gu4bSR8icVxnZbWP+5u86rikVltDYU7HaCVZZZAltWIREGsVn8ZbjE1UCy5ikDoEQTX6kOniMBEiABEiABEigtwnce++9KvBlY2MjmpqalPuGLPspFhOyz79ECYiJ/wZUYRvWaquG7aXY+JwZDbEI89aLC8B+fFvneaoRX/7ecpR5m53gVjSUVz2DudiP1/fphysrkOXcZUbMCm8d3v36bWp1j7nPea0mvBm5n5YEbgZwSFkvHEJALBskdsShHowDMbkkGKpAuYGUYZBhQdFXzNJ3lY0eJBRQwSNeQITVQ3uwtR6oylFGdNqVXbOUEqjGk/e7Y0HE11obDrzz1/hg4Nfw1IMPprnZl60Q0CtURAJRUZJwX+vkDb2iCOIAWCRWEtNWIWqTsbTnrK6xCtPGrHJL393VNpRFSEXCfXYLwz0SIAESIAESIIFkEpCVOOTvrrvuUhYRsgRoW1sbLl26pFbkSKZsKdG27ZrhL0to/Ad3PkvhMM9IlFUuJLaE+VdetRWvVRkp9VZwxVlGUqhLxXCMFeuKMxKwrQj1NS9h35xn8FoMS5GqaiWQ43f2Q5YFjWX5UlMUbqcHgfaTh5C/YCmWDgrgUO0OHIIs3zk51B3D6M4giQMYg5nDoGH5wKHd2GHHxSyZuQQzSwbBXqjHqLH3N6mQsJcPXfqCf5yB3h+COFsIUUbo8t1XSnxw8M+wp2MJHluU7soIi4kVI2KPbREQGvwnWowJTTamX728qLhs2FYSddVjoBb5iKkCd6a67aLYkNVBahyrB5XDbkdW21jCZTrd0LhHAiRAAiRAAllIQNw3ZJlPWQZU/kkwy6FDhyoSopzgnxCwXTPMx0HlynE3vmymxQBLxZCY8AA8cSxdJd0BMS3FgytDyE4DDspylXgJj6pfI8OpdXj03eXYuN6wnFBLi0qsiQ14IQ73DqNWbqYBgfzJS2Gt6FmCmUtLMDMGmS1FQztuzoRbcXGzHQFRaOjgEsoiYjIgVhi1hxA4VIvAIVF4zESJzhNDez2RJetdNtpPHkyrpT5/d6YW792+C0MHjPP5NxJXWo6gMYEzQ5b1fPWSKCP+Ko3jRrg7XrRktYr/gBcPhi4R21KLamsJDqzugRf7uoOqMjy7Wrt6xBHDwi02gDqo6uYvgXbXcLJUVKo+9fRqG0793CABEiABEiABEkg7ArIKhwSwlGU+hwwZooJbSoBLUU5k/Z+O83DIXqrTBqJWvphQGl6xoJaS9AaetANUzppiW5paS0n+SC25ESStlBbOqhpFmDprPNB4wROzzFz9I7gChI4xoVeMEAuI10xlBBrwI1necs4zVEYEkXNLE1DBK48j4LGSaA8cB0ruwgivsmGQKDuWYunSBZjsCXqpq+zt3yy/SwVwsLoOFZVlQf+Z3ibezfqH3rsaz/1Rdfh/Dy5CaZxttPzq+YxTRlgIKrDaiu6Ixa7VJepQbbtWPPvWangXQo0Tn8quV+xQLhySUrcdq6L6boRpqe6gtVypr2uHHc9BuW2EKc9kEiABEiABEiCBrCEggSslgGVBQYFSSogS4vbt20o50dmp3XuzBodPR4swb9kcYP+O4KoWdnBKV7yG+m1qBYxv6JgOSpHhDmppBZCcgy87VgmTUDnHE9RS1T0eVUaQyaL7HkDZqW2o1nVL/Epx0ZiwHF+K1UVD9awFe9db5TZWMYilz2AzaVAJKmaWuANUth/H7uPA5MklttXETQQO7cAOc1kNtVxoCfK9Cos+IJp+Lhvtdahea0T7r1sL5bK19AXULNBuFwHsrvoGdjgAv4Eq2alYjY2rKxzlQ2C35FmKF9ImeITToZ7bsFfUkK/yr+6s9dRbgflf+BZme1fh8ORK6d2K1Th3rhLVYxa7YzEYrhU9IX/Rku9ic+0erFo1DSrkg9R/dDOenrYKLy6uRmXUIJZaihbUWqYbeLbST1WiY2PswartdVjiBJ7U5flLAiRAAiRAAiSQTQQuXLiAcePGQZQPZ8+eVVYRsuTnwIEDGUNCnwjly/Hac9vw6FMrUGOnzX1ua5TYCxI/YgOwfh3WPrbNLjUHX391OUwdgsSP+DpW4NuPaV+L8aj6nifIpCzX+T3gG0+tw6Nb7KomeNwwtKwRfi2FiGSQQJtapsTri9BUeh2yXVhOaqnFzUU4S0wOR3Ej1iwvYZ/Oo91jEhgHp4oU3RhUMhNLcAi1O/TbsLhiLDVcMQZBYkbgUC2cLAAmLzDz9F3ncvDyO13S3KmvzAhp9WJTE0pL7gpJZwIJkAAJkAAJkAAJkAAJkEDPErh48WLcFdbV1aG0tBSXL19Ga2srZIUNUUiI64ZYT/zxH/9x2Dp1QMywGXiABEiABLpBoDFwFiNGjfKtYcK/HVbpWe6y4cuGiSRAAiRAAiRAAiRAAiSQ8gQ6OjpUAEtRPoirhuzn5+dDgllKXAlRTPCPBEiABFKZABUSqTw6lI0ESIAESIAESIAESIAEwhCQQJajR4/GnXfeqXKIYkLiR9y4cUMpJMRtg38kQAIkkMoEqJBI5dGhbCRAAiRAAiRAAiRAAllDIDc3N66+ikWEKCOkXElJCWbOnKmsJCZMmIDGxkblyhGuwnjbClcP00mABEigOwSokOgOPZYlARIgARIgARIgARIggR4iIO4WsQajlPgQV65cUa4aYikh5SRN4kmIhYTsy6/3LycnRx2TtvhHAiRAAskmkH6rbCSbGNsnARIgARIgARIgARIggV4gIHEfZPnOWP+Ki4tDssoSoPfffz8WLFgQcowJJEACJJBqBGghkWojQnlIgARIgARIgARIgARIgARIgARIIAsIUCGRBYPMLpIACZAACZAACZAACZAACZAACZBAqhGgy0aqjQjlIQESIAESIAESIAESyEoCH1w5j/997jAG9RuAqpI/RMlga/WMrITBTpMACWQFAVpIZMUws5MkQAIkQAIkQAIkQAKpTuD/nPslmm5cxsfXWvHaucOpLi7lIwESIIFuE6BCotsIWQEJkAAJkAAJkAAJkAAJkAAJkAAJkEC8BKiQiJcY85MACZAACZAACZAACZBALxD472O+gFGDh+HuzwzHo2Nm9EILrJIESIAEUosAY0ik1nhQGhIgARIgARIgARIggSwlcO+QYnznnv+Spb1nt0mABLKRQA5efqcrGzvOPpMACZAACZAACZAACZAACZAACZAACSSPAF02kseeLZMACZAACZAACZAACZAACZAACZBA1hKgQiJrh54dJwESIAESIAESIAESIAESIAESIIHkEaBCInns2TIJkAAJkAAJkAAJkAAJkAAJkAAJZC0BKiSydujZcRIgARIgARIgARIgARIgARIgARJIHgEqJJLHni2TAAmQAAmQAAmQAAmQAAmQAAmQQNYSoEIia4eeHScBEiABEiABEiABEiABEiABEiCB5BGgQiJ57NkyCZAACZAACZAACZAACZAACZAACWQtASoksnbo2XESIAESIAESIAESIAESIAESIAESSB4BKiSSx54tkwAJkAAJkAAJkAAJkAAJkAAJkEDWEqBCImuHnh0nARIgARIgARIgARIgARIgARIggeQRoEIieezZMgmQAAmQAAmQAAmQAAmQAAmQAAlkLQEqJLJ26NlxEiABEiABEiABEiABEiABEiABEkgeASokkseeLZMACZAACZAACZAACZAACZAACZBA1hKgQiJrh54dJwESIAESIAESIAESIAESIAESIIHkEaBCInns2TIJkAAJkAAJkAAJkAAJkAAJkAAJZC0BKiSydujZcRIgARIgARIgARIgARIgARIgARJIHgEqJJLHni2TAAmQAAmQAAmQAAmQAAmQAAmQQNYSoEIia4eeHScBEiABEiABEiABEiABEiABEiCB5BGgQiJ57NkyCZAACZAACZAACZAACZAACZAACWQtASoksnbo2XESIAESIAESIAESIAESIAESIAESSB4BKiSSx54tkwAJkAAJkAAJkAAJkAAJkAAJkEDWEhiQUj1fOANdU/IAdOLAvncx54gpXQn2PzERswcDlxpPoHB7wDzYB9vB9v0aazi2F+W7/I70dlpkudB6Hjk19b0tRLfrv3/sSIy9dRm1gRvdrosVkAAJkAAJkAAJkAAJkAAJkAAJpD6B1FJI7DqMtfmzsLE0F7MrK7DmSB02aYYLxyhlhLxg970yQgshv15lSTnq1xVj0pR5qEeylBJ+cpkyp/b2/Dm/h9d/bxgKFNv/wJwjVEqk9ohROhIgARIgARIgARIgARIgARLoPoGUc9nYtL0FDdKvwYX46kLdwXLUa8uJX6fa1/56/LCxUwk6aXS5FrhnfqdXoG3dPLQtK+mZ+mKsZUvVPHStm4EtMebvTragMkJqycXsub+P/dMHd6dKliUBEiABEiABEiABEiABEiABEkgDAimnkADqUX7sqkI3qawCawCsWVaESRBXjdMeN47UInzp6pWeFWjEQBT0bI0x1FaCseI10wd/bmWEbpBKCU2CvyRAAiRAAiRAAiRAAiRAAiSQyQRSy2VDk951DgfKJF6EWEmUA6W5AK7iDVfciNDYCe7YEvq46WKh0wAn5oMdt0LKrsc45S7iHNPyRPwtx1d95QPE0mDFcKOwN56DWEDMLTSUDkFZzbIFpRPRtW5iN2Nn2K4luIqtx4AVyuLEks3pr0uePKxYNw8rJP+Gw1jpy+kEmtQ4eeJ66FggN9qw9hXD7cZA4a+M0BlspQTovqGJ8JcESIAESIAESIAESIAESIAEMo1AClpICOIA5hxswyUAk6YU29YR57DSoa8VC/ICvxc5G/ZibWMn5MU9YfeGEaKM6MDWDdHiQMjLsrg06H8in/3S7sinlRFB+XLE6mN4MbqqbLcO/fIvL+0bpA/n0aBcFixXiZU1e6HKKMuQE6qPkWNneOWy5Kt33F60cHlYMfqyqs9qUxjb7hlH6lC44QQOqBAO0ieR67DBHYCLU3CcCkYUKWsWaWXLaMvEouFkIsoILSctJTQJ/pIACZAACZAACZAACZAACZBAJhJIUYUEgCN1eKPVRn6jDetN6wgnwGWL48KhY08UlI5JKPZBAa5grffl23fEDSWDemHfi62tljWBowyZXoFHxDLixhW8oVcK2XXZio0xfFgY+epR7qcA8JXBLzFULlHUhK78cRVbnVU3rqBJKR8GYux0vzpD00I4HWnBUalj8EBMU9m1y8dVHPZZdeTu6RV2AEu77k+v44zloaMSLl2+iianWSolHBTcIAESIAESIAESIAESIAESIIEMI5C6CgkAR69awSJx9XpwtQ2JKZE/UA2DO2ZD/C/XrrH0tOE6FmVn5a8taw5HGaJjPwwuxEaXJYVUZL/8Oy/yZp5Z2B+jYiCKSL13OIRTAG9clHHKwwyxxphehGkSk7L1stuywpZoQdmQoIvKp9fxxr462yLDznD5HGbsa3MpJSaNG9Z7/WHNJEACJEACJEACJEACJEACJEACSSGQmjEkkoKiG40euY4mVywIuy5vzAhXEwHMeSXgpKxZZi93OncW9uNdx/LDyZDCG5tOXsH60kLIKiNr8i2FQ8MF/9VQ/ueu03jkzyZgwaAbeH3XUfy3hhvY4lHCfHykDjNQgcPCtLUJa3/SnMK9p2gkQAIkQAIkQAIkQAIkQAIkQAKJEEhLhcSm9g5sRC4K8oYYfR6CUWq1yA6c0W4SxtFe3Zx+B0aZDVzswCXkoSDvDhVXYZM6ZgSV9HEN2bT9XUxTQTBzMWqEWVkabIu1R2UhZg8vwvo8KwCpn7uG6kl7AAs3BxUx4XonSonRfT2O4YRhOgmQAAmQAAmQAAmQAAmQAAmQQI8TSGmXjbC9lVU4JG7B8CLHxSG4NKgOfhmwYxMYL/g69kTYihM5UIL9ldZKGZca7bZ1/Au1SohVp1c+sYhQgTF1kEuEj73gVrwkImMiZWKPKyFBSC23jVwURHDXSEQKliEBEiABEiABEiABEiABEiABEshMAmlpIaFW4XgF2P/ERGvFi7nW4DjLV9pjtbLmPGasK8akKfPQNUXiGpy3AlCaS3HGPa72ahZ2m7p4aNt7AbF40G3bq2XolTLEIgLKTaMYXeuK7Wo8q3XsOoyto2XpUCuPe1lT3bL+9ZdL+pzjBLHUeSP9ysoZRWo5UrWayFyPTGGKSlDRrwprAOHcNcIUZTIJkAAJkAAJkAAJkAAJkAAJkEAWEsjBy+90ZWG/2eUeJxDZJSVSc08vm4WXSsXVA2j6zTGMfoMxIyLx4jESIAESIAESIAESIAESIAESyAQCVEhkwiimQB90UM7IVhwpIChFIAESIAESIAESIAESIAESIAESSAkCaeqykRLsKIQQWDgDXVPyLBat56FdUgiHBEiABEiABEiABEiABEiABEiABCIRoIVEJDo8RgIkQAIkQAIkQAIkQAIkQAIkQAIk0CsE0nOVjV5BwUpJgARIgARIgARIgARIgARIgARIgAT6igAVEn1Fmu2QAAmQAAmQAAmQAAmQAAmQAAmQAAk4BKiQcFBwgwRIgARIgARIgARIgARIgARIgARIoK8IUCHRV6TZDgmQAAmQAAmQAAmQAAmQAAmQAAmQgEOACgkHBTdIgARIgARIgARIgARIgARIgARIgAT6igAVEn1Fmu2QAAmQAAmQAAmQAAmQAAmQAAmQAAk4BKiQcFBwgwRIgARIgARIgARIgARIgARIgARIoK8IUCHRV3V1Gd8AACAASURBVKTZDgmQAAmQAAmQAAmQAAmQAAmQAAmQgEOACgkHBTdIgARIgARIgARIgARIgARIgARIgAT6igAVEn1Fmu2QAAmQAAmQAAmQAAmQAAmQAAmQAAk4BKiQcFBwgwRIgARIgARIgARIgARIgARIgARIoK8IUCHRV6TZDgmQAAmQAAmQAAmQAAmQAAmQAAmQgEOACgkHBTdIgARIgARIgARIgARIgARIgARIgAT6isCAvmqI7ZAACZAACZAACZAACZAACZBATxDIy+2P52eOxezSO1EydFBPVBlzHcVD+ra9mAXL8IynL1/HntOtWPezk7jScSvDe5s93cvBy+90ZU932VMSIAESIAESIAESIAESIIF0JjB95FBUP3gPSvMHJ6UbVEgkBbvT6EeXruNPa9/D0YtXnDRupC8Bumyk79hRchIgARIgARIgARIgARLIOgJzSguTpozIOtgp2OHxBXfgTz43IgUlo0iJEKBCIhFqLEMCJEACJEACJEACJEACJJAUAl8qK0pKu2w0dQgs4TmQOoPRTUmokOgmQBYnARIgARIgARIgARIgARLoGwJ3DOiHccOS46rRNz1kK7EQmFh4B3Jiycg8KU+AComUHyIKSAIkQAIkQAIkQAIkQAIkIASuf3ob/XL4KprtZ8Mduf3BQIiZcRZQIZEZ48hekAAJkAAJkAAJkAAJkAAJkAAJkEBaEaBCIq2Gi8KSAAmQAAmQAAmQAAmQAAmQAAnQTiYzzgEqJDJjHNkLEiABEiABEiABEiABEiABEsgaAnTZyIyhHpAZ3WAvSIAESIAESIAESIAESIAESCBGAh23cOzobfz0t13oyM3BjHH98MXx/a3CN27hZr/+GDQwxrqYjQRIIGECVEgkjI4FSYAESIAESIAESIAESIAEUonAhf/owLHSXDw0KoJBf1Mnvnu4Cz++rSXvwo+P3kLFyVt4sD/wNnKweZ6tnNBZ+EsCJNArBKiQ6BWsrJQESIAESIAESIAESIAESKBvCdzCiRbg+bOd6Jg+AA/f7eedfguHjprKiKCEdVeA1lzgK3cB7568hYfKElBKvLcZC57ZHazUb+uex7Ht+wsx0u9Yyqc14+2v/SVe+XAintjyj3i4xzqh6wUmrf5XbFzUYxWnPNFsF5AKiWw/A9h/EiABEiABEiABEiABEsgwAt868imONffDqmkDMNR0vWi5jbevhe/sx53Az27k4PnPJ6CMkGqnrsLunavsBoIv2YteehNPTg3fbsYd0YqZhf+A3U/cF0P3RqJkPIAPY8jKLBlFILUUEgtnoGtKHi41nkDh9kAI6C1V87BieCcO7HsXc46EHE7xhBLsf2IiZl89j5ya+hSXleJFJDC9Am1zC9F0bC/Kd0XKaY/5YDvPjTasfaUOmyIVgacMrmLrhsNYGVIm0XxAQxS5I11n1jEtTB9cizGz1jLF+Ntb9cbYPLORgCYQvKbCXes6J39JoLcJGPNKtPnKfl7TEkWbV1Q+T5nIz3q6Zv95Jnjd6Hzy659X54g0tyFG2XRd4X/LUb+uGJPMDK0Z9NwX09xpn0dJe97tj4l33gIuWIPw47O38eOzHfiTkTlYNrE/xhZZFhMd5hh5t4f1w7dnptYrklfE7u/bihJEsdRo3oW1K38A9LHFwvi7aB3R/TFOnxpS62rbdRhr82dhY+k47J8ecCsdplfgkbRVRvTFCdHDE0BMk05f9Ctd2zDG4xVLASUPQxvXzcA0XwWD9NN6kBnVeAI5tkJuzbJZqgxcZWLNFyoD1LjOwv6LYZR6C2dgxXB/5uphLq8NazdYShUl29xZ2I8wdflX0yupjmxRFT7da76v2olbSvUwPTBNlbVx9zZzCnBei38sea7HzyzWEgvHYPbgGBRjzhjstZ7T1LwyD235/h+TpHk1X5R2YOuGvbaCXeaxiWhbBuMDlDFnbbA/3EhbIfNMCcbmIezHK9/uRpjbLGWE3D/N/nhl863Vnai45Cmlf47zscLqU9e6YWE+Lrir6NO9jH3O68Loqf2w9pPb2NgZJPrj5i78uPlTlSBeHL8LHgrd8vPyCM2V5ikXcUosEe6J0o3m02gA3Eq2KEWcwy5LESeVGyQQQiDlLrlN20/jwI1czL6/3BC2BPsrC1HQ2uJWUhg5uBnAnFf29pz1xZE6FG6IZgFA6mEJ6Ac7wxpmZc15NCAPjywr8S22ZlkRJt1ow3rDOsi6HtxlYs0HHxmgxvU03vCVoAT7y/LQ0NiGS97j+sXpYNDCY9P2FjQgF9PK/PvjraI391fW7EVOLysjRP6+aiduVrsOI2dD8hVDccudpgVEMdUW5jqOq0sjBqIAHTiTdhZ/cfWyZzPzXO9ZnkZta/IHAjc6cNRIC9205olLjaeDz2NH6rC+sRMFpWOwJbSAUrZ/tTQXDcdMa796lB+76i7jN2ftOoytrd5nQquRpvZQS1rf5sXyMNzcZh/z9ueNVqBgRBHW+FcYmiov91MsZYTbclKezSLP/aGVdT8lpntUxj7n3cbPfn4b7/YDJodB+fFt4Ldhjkny5xL01IhQZeghcWdY9CUs+NouNOujfmlioRAun6Srf5vxnq5D/+q6/PKoY3+HnZL3wx9gubd+u473XvmSEwujofovVVtrdzrS2rnexz87bXwJruNahlfe11IBuj9Omb/B294qg7m5lSUEUk4hAQQw5+RVYHgR9k+3R8FvksqSAWI305PAltF5QOtlj6tFPQ5HeMiZlpcLXL3ucekI4MxV94NRrPn8ZRCeAWzyeQFas2wcZqMNPzzpw9z3xekKmm4ABXlDfAowiQQylYD1dTZTe8d+kUBEAtOLMG1wJ46edCsDNp28gkvIw4yFPqWn34FR6ETTRc+xix2uMuEUIkevdgLDhxnKjiEYpV0hPVX67Uac22DV5VVuqDYHD8Q0vwp90rbcX4iCG234oWMZYWaqR/mGvVjvN7ea2XpsO9vvUf1R8Vmg7iZwPEGmFcP64PVo6p/iCbFO+PC0o5BoPvuRJfGHe/Gf9kt685G9ykJh0Uo7AKa85D+zWwV93L3zTeze+Q9YhN3420WGUiJaHmW5IOXEQuJxbJN6fAJsTn3iTex+aYGSSYJMSnvuQJMn8MrKn2O2kuNfVX8aqjeEVzDY7h8Nuk0lu9QRlH3khIkAJmICPTYSPHvTs1hquWxohqIRHz0PKyorsObIdXzVjivh8qO3Tc0KdBmvj54ynYPbRM6bpuoYgqP7WjBqbrH6Oh3Wx982xdPNwWzPqcf4QumXZhe2TBdzrT2vn6aW8RiwYkqe05zyz7xoxS7QfXb7bBqmjvqrvJeRb1vBNkL75I2TYLfhPAh4TDt1n49dwbQphdByhvMTdToXEjch1A9UmcobrgQhdXrGx3tclcd5rL1ahI3qS03Q+sNdd2jblpuD7k8nDhy7EhTdd8s2J70Yms96sIrwkJN3h/oiE4wzEebBImq+oAyu8y1cTIrpFVhfChzYJxYQFVjv6deai+eQs8H9AAr7Qe6STz89xQE9xnDH0bDYu88jJe+IK554G+5zz3d8xZ3EtJLwnBPwnv9KyMj1evuh5HW14y4fzYdZ6nOfb16zY6u+aRfP4+iIYszW15qv7IZ0qq+Gy0bEa3GIy8fZl6VxrVituMfIuibC3Ds93M36Q/kB+voKxmTxMvW0reoHtsZ9j5SeRK5byYfz2Ipiw3XJaN+8p5ZORNe6iZHNxqOxsO9pK9bNwwrfa9P2RzfnG30OmeehKZd005PfMkmPMh/abKZdPIE38iZGiNekz1Gdzz4PveeoHqd9HXhkrlg4Gn704bjY/QieC1bdrvNGlTXOdZUl8rgGz9coc7RHrhCOdledn2j5veMSMsaxyu3zjOKt2+TrCOjecN97zLnOlKMQG9fNw0bveOqqfJXTAI5cR9PcQozKF4s571whhXMxaoSuxP5VdQFNZrKPEkAp4eH1+O9EU/4MdK3TzzBmf4wKo8xtmH4FP9yw1/MhAFBt3rgSxVpEt2PP0yEfFPRx63fTEZOL9zoy5TfHQ8oa9yBVlX1fcKo3yprnRbR7lO/1Fq1t3ag7n3mfl/E/c1VeKpP3N/Tz/fGt5lt4/mZiMszwXZkjsbrCl9IBHHfjwHurMHVqM/7zpyeAeyZi0ocncEoUEiOB5lMnACzAbBUMsxlvb5HVOybioen6jf0+zF4I7NwVrCd6nvBSxXdEVtlYBStOp+5PUPbodd2HJ3e+iSejZ2SODCfQByrAxAiu/HUbLg0uxPonQs3YzQfYnA17kbPhBA7kFaOrynTziLXdXMyeOwyHpR7zZcYsLjdt2xTPau88GoYXo97vS4BZzm97eDHW4zQcuVGIjU9UeMwC87Bi9GU7z16sbezEpCnz0FUJrFf91WkzjC8G3sbKUT+3EJB4BJoRCrFemxnH3Sd78pEgRY4MA7FinVeGXMwuC8qZo0wyJSaIV77g/paqierL/FqnXmC2KKPsLNZDlEzIMtZ7kbOvDSidGOTvPKDaxzecR1Op+H96XAkM9tqk0nnQ1XUf68Bs8VfV8hoTttXv00CZVk4E++Deiu/rjS7rnPOG3OrLjryUGg9pseWzZRgxzjjf9mJra57PmFkuUTBNcLVQ9q/7QcpK3FIlgbuu4g3DxcRTzNi1LZ+MfkjMjBnqhWwgxmreKMEjI3Jx6WKL6wFx0pRxwEF7fGM4p3SAMlHamef/Rs89Iu56jR7JZrRz15Nd+VGvGG6cy2H6UlBaFOzvhvNokHuhcV546/Xf97sW5SXavt/JOR+mfRjXivDzP2987p1RrkXr3B2CR5zxBtaUDTG+LMZ6n0nkHhlj3cOLMeOCcd7cyMMKfd4oE+cTOHDDUiQJG78AzGo8orEQN6NjV+2XDWnPNGfXI1qPHzZ2ArYC0kq1rpuGk7b7VMg9qjvzIVBQOtHuv/HyrsUxfoP5RPYTOBBuLtPzllaUR+JypA5iKj9ptDmXe/pryGBtxjiuIeU8CfHOidHy2+MSnIPt68iZ72OV2+c6S2DMI891lsunPGsoxa3cG8I8DykrBg+6qLt6XKeYzwvlqLc/ulhKDMByAzSuN6nYif1gzBPK4iIXs/PM5yS4524lVPS5DUcCrrlGFbPbdK6xqB205ttLV0M/QkQrGryO9PUW7byQ48WYJAoo+7lFubToZ6Z47lEhwkVrO1jANXfaz2TeZ65EeARb6OZW//54aHZ/bB4JzIizqs8V9kOF/qIWZ9l4s0/9omV98NHZZqC5Dj/9EFi0cjlkoYmdPxM3h/dxQKxuFj5gv/TbcR8gVgXaXeNL+FvbMkfVg1jyxCtpD+UfWYGHlFWI7Sai3DbostFDdNO6mpRVSIivu/JJHJwL76QQahrn4+YRx7C4/RrDFbyKw44pnmV+98OLnhfecEXN9NbzxgNsAHMOiuLF/YCuvrL+OrgSx6Z2+TLQiQOm/75KMyv22zbNKq0HDrfJYBx9Um4znThgyqViCLjjG4gUrvHadVnFGQj5MuIR13wB3bT9XeQcbLFz2A+jpu+py+/R8g2FK76I9RAf4s/qic9gvRC7uWLXORXDRMdF8D3XZMw88vfIrvTLnti71s2D/BPllfWQaPj1xpoPQAGuuGJS+MaxsF2iYlMsWD0VKwYJfhnbtWPTUeeCYdK7cJiySjrQasShCGMO3HBMP6wBiOGcstxVzhuroPjHWIm3Xr9xDn/uhuZW57b54hmuL57zWbn6JOAa47oWlXm0Z8zstBBJPdeKPm+85tju8Y/hWjzSgqM3jPH2KqDU+F/FVv3yiuBLirttz70olntkrHW7zK4DeOOiVyEQQssnIQYWPqX8ktRLmjlHyHWD4L3b9x7ldXv0qzhcWqt53YTLZFlhaMWucrUMN5cZ85bUJtemy18f9v3a9tdfeUFcNg3zfE9/QySKY24KKRuSEOQKxDLPh8+vFW1mTCBXDJo45HZfZ7YSzzVO+hnI4Gb2TcUA8s7N1lznjtdlFurZbem7pdi05jZLMSqxFYCgu4Qwtz746DmwSz7OKMWdEWvFfgYwVyzzjT+WwNymPniJosTFNxEW8nKv+6p/TYWMXae3naj3qdC5TF0zLmV/IvLKS68ENfXcW8M955lzsn5ejyfmRoIixlXs2m283gwcjqsQ8JUpAzAozjIJZ5/6gHKbaPhpHd5TrhliCWFZPGDXz/Heez9XcR4mTfCaFy3Ay8pNQlw2gv/c7hSx5ElY8gQLjsTD3w/Ku221WNKIciWolBi56B+xe+c/4mFtAJJgSyyWXgRS02XDZigv4htlsnL5HYYxjbN9EaO9+CY0PLYpomVWC+sLgnw9cJnfxVazv8bYx5QxpursLwY+8QAA8e8vxuy589A1VyqzTf60zHH2yfoqYjwQKPmsGAKTEnhRMrsnbgwrbNNCSbdcUeyvFuphtBMHXOeAWdr+MuFxG7DOnUh8ZPKVB/tcTHIYBeu1FA5hzjWbXTC3d8viMsqbHMu+PGh5xlO+wof8xZovxITUO2bWVyph7nKJCmnQSFg4I8TtxTgaYVNiaBRjhfr6WW+9mFw8hzntA9FVJsHDAoD6Wn4Fb3gYRKjU55ClxLrUGP/XKp/KIiZFPHfDlHS70FiZ5KE8eX8+95+Q88aSLrw5thyP5VqUF/xxmK0eXAPY5FFAqZc45MK51xpQEmMUvAf0fN2GcCGbsbAIKRQmwbpuHpHgsUcC6rqRlyXreg1zj+rN+dCWMrG5zLo2C4Zb7i6uDt+w90RJN6XYikWwy1JgBPvrKqF2emxuinNO1C4KzrmqXRzUHGtZeoXGBArKn7jcdt2Di9G1rjhYodoSixufP+Ua0YmjrnnUNqsPcf/zKW8k6ecyIynmTVFKuOYZsfSQZzxXDaKUCH6QkUNyz0SIy4arkHITUW4CTn8SmNtkpSuve5G3Gd99vzlfFAdBFw3rvu9b2JUY233KtmTQLn2qhjBj76o98k7s56RPPBCpuieUIpFFjO9o0QA8P6ETT58C6mIs+eBd/fHFohgz90g27W5xGgfGi7vGPPHSwEixnNi1GzVbrHgKQfeMEZigLAw+QqAZmGq/tEvwSbGSWPTSm3hyaix5ekT4blciyoeXT4ns8bh5dLtZVpCCBFJaIZE6vPQEqScBy8dyfWP4Ja7ikz3MzT2+Sjy5g5Oh5fIgpvrzsMLxM+3tPnnEibArX41VzARlyptnuadM8fpMRqigW4cM38uQehKwgFF1WA96s30UNfH5pUplYV44fGSVZdCCD8C24iEknydBKWUAiEvQFPcxpcy63/D7lsO2mbL4iwa/jrrLRdqTLzkryiROhryYdOLorwPAkSFoWFekzPjP5IW6a0SqL9nH4j13g+5H79oP5pYvcLL7EWv7wS+ZsZYIzSfB79aX2lZhvgqo3rz2e7Pu0L72VIp13YjSbohyc2o45n5h66l2ul9PbHOZ29/c26qpuEQf9jfeOTFSfm+femHfmct7oe5IVSpl1xDLzc5UHCs3CtPaIVIl9jFfRUloufjnTf3BIY65TcVQsWOJGRZaodL4pdhzvla0+mWJKy3SfcqIK2MvKa5jxMTVRG9kNlZoCVE+9UZ7UevMwdD7B2Lz+E9RV38br58F3o1QZnJhPzz/+32xvIZbCOW2sWs3du4CJq2uUAoJKMuJ3dj5oSgpHsfvOdYCI/HwygV45ZndeGXH+3j4ifvUqhU1YsF9z+NYpoI5xJLHkMEOquk0YRwyNxtOiUYzWi6zROh2886/wfLqE8DCf8BukR3NCKg4nkaMjK/9JV75UFj8qyeAZmh9TMkcAqnrshGWsdz44fGpBWBPbG5rCncllvbXnRbfnm0qp3yqo6wuYAdr8tYfsiKByue1PPCW6t6+MhPdYMWdcPsiS72x9clyGzF8OJVIiftN+vZILedmxYi4BLst9fDj8wXXqcB68fZy9df0O4Wsjah1hznX7AcvT22uXSt4pdd81v5y74mPoAvKF5SukJgcFuOGC8EXkNjyhTM199SnmesYGvK7T1xSRFHjWUbW8IkO6zevOxPuV75+ivn5MnHX0JYQ8gIiZvwVlpLCE709XFXh0/3PifD5e+CI5qjYea8TXb81/i7z6xjOJV26T39dMQtEERUmSr5LKH/uIdei8ieX8S5X8UJMtxL/+4yrkYR3erPuUKFiZBFa0D/FvG4Mdw0reFxy5kPvPdeag6PNZf5cvJ1WsUbEbUMUpi4XGm9OwH9cY5ibwszRsc6JQUn85tAwc0ewUOJyq4CBPmNu1B2y6TvXxars9tQmliQ+yz1bX/ZNFxaznLxEhy6Vq+4NzjwgSgQJUmnEcFJVWHKarnFqDnRiceh2PPn0PTmWuU0Hu/UEXdY1x/JruU0U4qu+scXKIcuexvLnfz4bJdVHBLdbW/efba36/dv2u5ZCn8mU0iiMZZ0hfd9v3rqN1uYu/Kw5sjJixmf7YeOsPnTVMEmMHIdJat8MVGlbOQCY9JCtpNBlZIUMWfli199ZS36u/AHUqhXmKhmx5MF9eFKtoCErdHiWH9Vtye/UVXhZzmu7Pdeynma+GLbFIkK5aWjZF4nyQVxLdGDMGCphlowkkIYKCUAH9Qve+H38ddUEbPirI/YJwTvSoS+A9oulDmAUMkHb8ngrkn1XMEwtt3d5SL+CcaaFTOxuE9KoffI2Z8dWMP1N1ywriiOoobdCvR/6oGI92NgPtjoYVlkwyKX6Su883GjfWWOZWHusLzWec5uH6ib1r67bDLQlL91O3UDoQ4YVJCtavCPLn9UdnGuLNwik3ZYOxuoX0EuV8TyQx5xPlmHzBEP0q0/jiPwbDJIaXhmhfWZ9/GSdyuWFJBezS/MMaw5AFDgFpYWGksIpkMCG3zlhyxbyEJtA9U6RKOeuk09vWC9jwYB9sZ1LunSf/nrPG7WknVYghZPEj7t13/Vei3JdFZTKKiKeFxh9n9EB2qQpdS+bFwxiG675aOk9Vrf1shnyQu5qP3YWrmJhdyylnbpuPMsJ9/V86IiY0FymuZhBoX2uTRVrJE+tNGW+iDptmxt6XO8PBsIMmZtimKPjnROj5Q8dF2uFHUfpHIvcZj+NbV23GehWrK+cuo28alPPdeY86hMvwFvMf78e5d5guGoli1yY17klj1Yu6LhORoBru4ypkNQxnMyg1lZgZ3fwZLXEqOceZeUz42b5S++X6gQnDhPI01qdJwJfqXTXYTsAue6zbsmyaBh1o1MnRP7V50W4e6D32dbm6K40lnuUu4Ta021HupbsYpM8z02PSEwp58OJNTd29eh86yNvmKSb5zvx+u4OrHqrA39S+ykefq8L28PgvzMX+Nb0/koZMbTXjCOCcROetJajcEs+ciE2qjgQZtyEYBl3XAi7qFq2MxiLwW/JTlEkmPElouYxFRpuCaGW/7RjVVjyaPlMmeHkc/qpZVDWEFalVowIQ3aXMkLX611e1CMQdzOOQHq6bIj/PGQJTMPM3Gu+KEF2ymZho2OKLss1tmHUlIFxD6KYZWPZLLUU1gpdWtpzVheQCXoYuqYE/WIbjp1Hw5RQR7RLjefRVCYTm12RV25df3d/5esAZqDLjI8gPq62KWL0PnkFkK9AwP4npI/abzWSWaG3fLh9MXsF6tcF2VlLJwYjzouFB6rmWUuR2dW4ljz16Wtkk+CgLLpuxw/YiWFh59F1m+fRvvNomKsZBOtyb/nwEv4b7Mj47sz2nmZh+AZrv2RX/hjzOdeJwda3PlflvjuWMgVqdRNZ6tD155zD9peUiIogO4aALL3qPLwAlhl/IQp66guLHjfz/HfkdEnfjR09DgZfZVkSPHfdlUsQ2yK0zQ2Or5zHa/PlPjUDW3aFK+eupU/2brThDYxTy1qq9mI9b3y4+16LKk5AHiZ5Xq6tr9PWfUYtPagat6x15pjm4QlBCF6T3a1bXggf0eMY7ryKlUWMfVFuG8PzXNeNKupc530zHzritp7H4dEJzGWai3NP9VmmFPo+ARyNajEVHNfwc1P0OTreOTFqfr9xcc0Bscjt0HZvOHUb9x5X3e7ssqfnOve5bwQLDi0SPkWPoXF/dc3JPiUdXhHLBJkE5fR5zvDrv46T5dN2xCRnFQ97uVNXZp+2XcfdO9LHTcqSUMfuso/LPeKCPCO68/vv+TEw74HeZ1uR8QTGPjFRrQa00n7Gi+keFSJAsO3w15IUkufpDjyiliu2Kok2/iFN9WLCoOJcfLn4Nh4O3Mbh07fx098Cx24Dv7XbnJwLTByWg4fG9UNFSa9pIXqxh6yaBDKPQA5efqcr87rFHpEACfQ9AfkqUoSmfQk+5Pa9wGzRh4CKc5HXhrVhvxb6FGJS7xNQMXaAreYqLb3fqk8LdiwlWQI6bl97n+qYRAIkkBwCthtm07G9CcWESo7QwVbPPf6HwZ0kbBUP6bO1OJLQu/RpUpa/5V/6E0hLl430x84ekEAGElD+3tHM+jOw3+wSCfQ6gV507+t12dkACZBA6hKILRBt6spPyUiABDKBQHq6bGQCefaBBDKNgDLjzbROsT8kkFwC1sos9nLTtEhI7mCwdRLICAK2lZMsG9p6Ht13h8sIKOwECZBAEgnQZSOJ8Nk0CZAACZAACZAACZAACZBAfAToshEfr0zNTZeNzBhZumxkxjiyFyRAAiRAAiRAAiRAAiRAAiRAAiSQVgSokEir4aKwJEACJEACJEACJEACJEACJEACJJAZBKiQyIxxZC9IgARIgARIgARIgARIgARIgARIIK0IUCGRVsNFYUmABEiABEiABEiABEiABEiABEggMwhQIZEZ48hekAAJkAAJkAAJkAAJkEBWEDh16XpW9JOdDE/gw99eC3+QR9KKABUSaTVcFJYESIAESIAESIAESIAEspvAmydbshsAe4/tDc2kkCEEqJDIkIFkN0iABEiABEiABEiABEggGwj86P3zOH2ZVhLZMNZ+3ZXvVAAAAkdJREFUfTzZdg3/XBfwO8S0NCRAhUQaDhpFJgESIAESIAESIAESIIFsJdB241PM/79H8erxC2j83Y1sxZB1/RZXnY3/2YjP/6//QOv1zqzrf6Z2OAcvv9OVqZ1jv0iABEiABEiABEiABEiABEiABEiABFKTAC0kUnNcKBUJkAAJkAAJkAAJkAAJkAAJkAAJZDQBKiQyenjZORIgARIgARIgARIgARIgARIgARJITQJUSKTmuFAqEiABEiABEiABEiABEiABEiABEshoAlRIZPTwsnMkQAIkQAIkQAIkQAIkQAIkQAIkkJoEqJBIzXGhVCRAAiRAAiRAAiRAAiRAAiRAAiSQ0QSokMjo4WXnSIAESIAESIAESIAESIAESIAESCA1CVAhkZrjQqlIgARIgARIgARIgARIgARIgARIIKMJUCGR0cPLzpEACZAACZAACZAACZAACZAACZBAahKgQiI1x4VSkQAJkAAJkAAJkAAJkAAJkAAJkEBGE6BCIqOHl50jARIgARIgARIgARIgARIgARIggdQkQIVEao4LpSIBEiABEiABEiABEiABEiABEiCBjCZAhURGDy87RwIkQAIkQAIkQAIkQAIkQAIkQAKpSYAKidQcF0pFAiRAAiRAAiRAAiRAAiRAAiRAAhlNgAqJjB5edo4ESIAESIAESIAESIAESIAESIAEUpMAFRKpOS6UigRIgARIgARIgARIgARIgARIgAQymgAVEhk9vOwcCZAACZAACZAACZAACZAACZAACaQmgf8PLiWiqI813ukAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TTA (Test Time Augmentation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TTA_predict(model, test_data, nb_classes = 10):\n",
    "    datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.1,\n",
    "        zoom_range=0.2, fill_mode='nearest')\n",
    "    predictions = np.zeros((test_data.shape[0],nb_classes))\n",
    "    for idx,img in enumerate(test_data):\n",
    "        if idx %1000 == 0:\n",
    "            print(idx)\n",
    "        TTA_batch = np.zeros((10,)+test_data.shape[1:])\n",
    "        x2 = img.reshape((1,)+img.shape)\n",
    "        TTA_batch[0,:] = img\n",
    "        i = 0\n",
    "        for batch in datagen.flow(x2,batch_size=1):\n",
    "            i+=1\n",
    "            if i>9:\n",
    "                break # otherwise the generator would loop indefinitely\n",
    "            #print(i)\n",
    "            TTA_batch[i,:] = batch.reshape(img.shape)\n",
    "        \n",
    "        TTA_predictions = model.predict(TTA_batch)\n",
    "        TTA_predictions = TTA_predictions.mean(axis = 0)\n",
    "        predictions[idx,:] = TTA_predictions\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n"
     ]
    }
   ],
   "source": [
    "drop_outs = [0.45,0.44,0.45]\n",
    "results = []\n",
    "for idx,drop_out in enumerate(drop_outs):\n",
    "    #print(path + '_'+ str(drop_out)+'_ensemble_' + str(idx)+'.h')\n",
    "    model = load_model(path + '_'+ str(drop_out)+'_ensemble_' + str(idx)+'.h')\n",
    "    predictions = TTA_predict(model,test_x)\n",
    "    results.append(predictions)\n",
    "results = np.array(results)\n",
    "results = np.mean(results,axis = 0)\n",
    "result = np.argmax(results,axis = 1)\n",
    "result = result.reshape((test_x.shape[0],1))\n",
    "ids = (np.arange(test_x.shape[0])+1).reshape((test_x.shape[0],1))\n",
    "result = np.hstack((ids,result))\n",
    "m = to_kaggle_csv(result,['ImageId','Label'], '4-sub_adam_cnn_2c_1p_2c_1p_valid9958_ensemble_with_TTA.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.995714285714\n",
      "(8400,) (8400,)\n"
     ]
    }
   ],
   "source": [
    "g_truth = np.argmax(valid_y,axis = 1)\n",
    "print(np.sum(result == g_truth)*1.0/result.shape[0])\n",
    "print(result.shape,g_truth.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9, 28, 28, 1)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAD8CAYAAADOg5fGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGfNJREFUeJzt3X2QVNWZx/Ffz0BgeFMZEERkELND\niCIYjIKaUAYUCwcGIySV8mVEDRGDlcUNScpFN1YsshZuYi2+sXGFsMQKhSi+USFILCpB0QTFFxbx\nbYguBgFBUWB4men94/S5t7tv357uPt09Pd3fzz9z5t7bty+c6dPPc86550ai0agAALmp6ugLAIDO\njEYUABzQiAKAAxpRAHBAIwoADmhEAcABjSgAOKARBQAHNKIA4KBLMd/skqoZFX171Lq2lZGOvoZC\noF6p13KUab0SiQKAAxpRAHBAIwoADmhEAcABjSgAOKARBQAHRZ3iBOSquk8fSVKkR4237fiujzvq\ncgAPkSgAOCASRaewbeFwSdLbDQ9526aNmRx6/O7JwyRJfR95sbAXhopHJAoADso2ErV9aFJiP1oY\nG7nUXv1BYF9krjlX22vb8nR1yFRLw3mSpMUTlgT2rd68pt3XT7l6asLvti4l6rPUdZZ+cCJRAHBA\nIwoADso2nbcDEVLiYEQuJp94oyS+cYqpYev+WGmtJOnimpaczvP08KcSfrd1KVGfpS6bwUTbHSf5\nXXLRb+0s4NX5+DsCAAdlF4mmG4jI1fhFZprMhpvHSpKq/rIlb+dGaivuuEyStGDh4rye19alJO06\ncoIk6Z2vH8nre8CN62CiNeVPZlCx0IOJRKIA4KDsItHpd7v1oaXy09qtkqQRj3wkSXpg9gxvX5f1\nm/P2PvD1XPWSJGl+6yxJ0u6vVQeOeeOG+yRJox68RZI0ZO3ngWOeXr004Xdbl/EmX3SDJDKMUpGv\nz7DtDy90PziRKAA4oBEFAAdll87bAYlRcQMS47q1Jhxj0z8pmAI2T+0lSVrftNDbNqDa3DExtaeZ\ndjPv2/5/W/0GU44eP+587QiqWf2yJKludXDfSM2RJA199jNJUvTVYKp+1iPmGFufti7j3fTI45Lo\npikVo2r+ntfzpRpM3D53hKT8dOEQiQKAg0g0WrynopbKI1gj55zplVNFL5I09rVjXnl+v9dDz9U4\n2kS+rXv2tPu+PFq349j6TFWXw1ffLEmq/5EffWaTWVCv+XV4mpnilGow0ao6av7LUw0m2mzyzevv\nC339UwdPkuRnH6kyDx6ZDABFUHZ9opkIiz7jbfrBGP+XVeGRKDqHDT+7QJI0/+FgXW6f9oAkqfHn\nl3nbMsksUBjp+sEtm02m+iwP/av5+XHTYUmp+8HnrfueJKl+g4lAXUJuIlEAcFCRkWgmdo3t1dGX\ngDzqtp9bO8tJJtnk1T+YK0la93BwAaLk7MMl8yASBQAHNKIA4IB0PsS0mRs6+hLKw9izzc9NZkCn\n+sQTvF2RvmaayY7vDpIkDf7lC6GnaZlipr10f/rlnC6D7pnKU6wuHCJRAHBQEZGojWL2DTf/3Kq4\nu0AH/jox+oleOFqSdE6Px0LPN2fnRf4vRxiwSKd5ak9JUqRhnCSp/7n+g8aeH7ky4dhxe+Z45fqZ\nbyXsu26AWVvyX4Z/39tm6zG5DlMhs+g48dlH66efFe19i5V9EIkCgINOFYnabzTblyb5/Wk1e8x0\n2eQIRvKjGLs+4bGoH4reOH1SwrGX1pqVsy/vEfzGvHd/vSTpw6tO8ba1Hng/y39FZdnWdL8kqS2D\n6cwv3hl+m571ytxFXtnWY3IdStLen9RJkm5eajKKVPWJAon1g+9oMFlIfPZxws2J/eCpPrefXGgW\n+knOIKXMsg//Ft9FocfY27XzgUgUABzQiAKAg9JN5+3UGAXTguQBiWx1jfirw/x26HMZv+60rvsk\nSe81DfC2DVuwS5LUduiQ0zWVm7UfmXUaWzO4Kblpx0RJ0n+c9rS3bcXnZ0mSBnU1qd0VPfcFXmfr\nMVUd/u6/TZdLchpvu2Qk6d1DJ0uSPpzY1VzrAe6Xz1XzgnGBbVubUnTP/DmDk8WedPz8YXNXUvxj\nQpK7cGzqbweEpfSDwsnysUYCkSgAOCjZSHTNqqVeOd2gRHIUkxzBSKmjmFxc2WuvJOkXIw542yI9\nTZQsItEEIzZeI0l6/YKlGb/m0v/8iVcedI8ZOKg+83xJ0ubl2yVJd52c2YrzV/X+R8rtNpuQpOev\nGiWJwcF8sAOIUm6f13jZZB9NG8357ICwlH4Q0ZuemMepiUSiAOCgZCPR6ojfvrdFW0OP+9dTzTfQ\nn1tOlZQYgVofHDfrCg7pElxX0Prq8jkJv5//jW1eeUnd+oR9W8Yu88oTlk+XJNVMoj8t3tBr3jGF\n99o/dlr/V8zPWa/4G2fZQrMkafPB09s9T/Nxv+/s9C7dUx5jswlJ+rdY3zb92u6y/bzarMPPOIZ7\nx4yMZR2ZZJCZjGms+qKfV7bTE/OZfRCJAoCDkn3G0gcrR3rlbPrV0hl9v3nKZ9+3/G/KHo+/lPHr\nf/LeG5Kkb3Y/GtjXcOqYwLZklfgsnuovmwiydfGxsEP0zFeedHr/jS1mdH1QF/95O5f//scJx9jM\nIjmrkKTRm66VJNXdtNvbls2obSXWa7Kq7n7k/9R7G9s9/omDfUP3jelmhufTZY426wjLOMKctcRk\nnMMWvCYpffbBM5YAoAhoRAHAQcmm8/HpQWRwrDM4TUpoVc+JpQB7P/U3xqYztB44kOIV2avuV+uV\no18clCS1tbSEHe4h7QuxfnBgU3I9fv25jyRJj60Y7x1ju2Uy6ZLpMtAMIq3evCb0mAlvTvfKNZOa\n2z2nRb0mKmQXju26ue02M/LYe8UmSdL7d/uT/bPpuknXbUM6DwBFULKRaDkiYslddZ8+ktyzCTs4\nGC/XgUKLes1BUvbhZR6Sc/Zhs46Dy8w5bz/Dv53Y1rXNOtJlHESiAFAERKJFRMTS8Q43nueVH130\nK0nSxE2zJUl13wlGqZmgXktbfJ2vf+DBhH3pMg4iUQAoAhpRAHBQsvfOA4VQ86T/yOVZGxslSad/\nYe7zb+uQK0Khxdf5x4vMOhpeF45y68KJRyQKAA6IRFGxWvd+0tGXgCL7fr1Zf7SuxT0CtYhEAcAB\nkSiAipHJ7dnZIhIFAAc0ogDggEYUABzQiAKAAxpRAHBAIwoADoq6ihMAlBsiUQBwQCMKAA5oRAHA\nAY0oADigEQUABzSiAOCARhQAHNCIAoADGlEAcEAjCgAOaEQBwAGNKAA4KOozli6pmlHRq52sa1sZ\n6ehrKIR81Wt1nz5eOdKjpt3jd08eJkmqvfqDwL7IXHOutte25ePS0qJey1Om9UokCgAOeNonSsa2\nhcO98tsNDzmda/KJN0oiSkDh8TcGAA5oRAHAAek8OlxLw3mSpMUTlnTwlQDZIxIFAAedKhK1U2Cy\nmf4iBafA2OkvUnGmwCC96XevlSRdXNOSt3OOX/SiJGnXkRMkSdvnjpAkVf1lS97eA5CIRAHASaeK\nRO0UmHxNf5H4FilXP63dmvD7U4+YjOOB2TO8bV3Wby7qNaE80YYAgINOFYkOeTp2F1aD23lsf5lE\nn1kpWHHHZZKkUQsXe9vGdWtNeeyoB2/xykPWfp6wr3lqL6+8vmmhJGlAtek/n9pzvyRp3rf9P/n6\nDaYcPX4852sHiEQBwAGNKAA4iESjxVuoxXVVmIatJiW76cT383I98Z46eJIkf+Ch64bXvH35SvdY\n7Se9w9PO88q7v1ad8pihTxzwytFXt6Y8RpLGvnZMkjS/3+uhxzSONt0IrXv2ZHWdyajX8sQqTgBQ\nBJ1qYCmTAQg78JA86CD5Aw920EEKH3io35Cvq0amala/7JXrVqc+JtPQaMPPLpAkzX84PBJFx+ky\ncEDovnTrxLbeeXLC71UbXs3vheWASBQAHHSqSLTnqpckSfNbZ3nbkvvOhj77maTU/WVD/2p+/mbq\n+d625D6z7dMekCQ1/vwyb5trnxmKr9v+Ix19CRUv1W3aqzevcTvpo+bH3Z+cKSk4RVEq/jRFIlEA\ncEAjCgAOOlU6b6UbgMhk4MEOOkgMPJSrXWN7tX8QCipfa12kErY2Qrx5674nSar/kb9GQiHuTiMS\nBQAHnTISdcWgQ/mbNpM5ah3FPqng1ovWOp1nyrTrAttSTVOUpKk9g6+fWqRBYiJRAHBQkZEo/WWl\nr2XKeQm/7xtu/lQH3fNCR1wOstD9GTNmsaJb5qtzxZvScK0kKfrqG4F9dpriDbdfJCmz23uf3PIH\nr5yvW33jEYkCgIOKjEQH3hsXzczruOuoBNUnmsnQkb5mgZcd3x3k7avZY+ZS1D78YvCFMTYCrUoT\nwEQvHC1JunnpY962y3t8lvLYOTsv8n85Qt94IaW6OcZKvkmm7o74v4HwhWWSbfrBGFNY1XGzbIhE\nAcABjSgAOCibdN4ORKRK/wb+OnEwwqZ/kvTsoXcl+emfl+6R6hVN7caTAtuuG7BEkv8Y5WNRU6E3\nTp8UOPbSWnM/dqoU/t799ZKk568YJUlqfSd+LdrDuV80MhZ/c4wVtkpXtuwg8bkL/cfG/G3eonZf\nV92/v6T8DDARiQKAg5KNRO2AhCT9fbZZscUORNTPfCtwfFjkIgWjFxu5SMHo5cs9dkuSdhzonfO1\nV6yxZ3vFHQ1m9nP/cz+WJD0/cqXTqbtGzEDEb4c+l9XrTuu6T5L0XpNZv3LYgl3evrZDh5yuCR3P\nDhLbqU6pxA8mnvbsF5KkDyd2zds1EIkCgIPSiURjUUxyBCNJW0a238eRzEYuUnbRi41cdtx1qbdt\n6PzwKTjwNcfde7e16b52j9/bavokV3x+liRpUNf93r4reu7LyzVd2Wuv+TnTXM/oEdd6++puMlkH\n68WWtl1z/QWDwsY3zunxmMLY7FKK6xs/kL/ntBGJAoADGlEAcFAy6bxNBbNJAyXpzy2nSipc+if5\nKeDgKzO/k6ISbWu63yu3hRzTtGOiV37jCfNIB3s/fPWZ/mNbNi/fLkm66+TNCtN83AwiXv77Hwf2\nnf8Ns77kkrr1Cdu3jF3mlScsny5JqplEOt/R4tdKsNMUX5lruvGORf0pUmGDxJ+29ojbmjhYbLvo\npNQDjJLbICORKAA4iESjmT6E1t0lVTNC36x5wThJ6SNRG8XYCEbyo5jRsSen5jtyiTfhTRu5NIce\nk866tpWRnF5Y4my9rv3If0BY/BSzeG8fO+qVtx4dGHrOMd12SpKGdKlJ2L6xxZ+acttt5p7s3is2\nBV5vH8l7cJl5/e1nPC1J+mb3o4FjG04dE3odmSj3es1Vl2FDJaVeLyHVNEUr22lsLkZvMllmbS8/\nErWf70zrlUgUAByUTCS6ZucrkqS2NE9JslFMuggmVd+ojV5yiVykYPSSa+RS7hHLBytHettev2Bp\nXs59114z9e2xFeMlSX3f8iPcHo+/lPF5DjeaPrdHF/3K2zag2tQ1kWhquUaiB/8wTJL7DRbZ+Ory\nOYFtNruMly7TTNbtlPeJRAGg0EpmdL46YtrztpC+NEmq7/ql2M/2R+JH3+8vSGCjl96PByNQ6/gu\nM7m/W2yO/Z2NN3j7bPQycdNsSVKdgituQxp6zTteuXGw6T9uXRx+O16y6jlx/Z97PzU/YwvBDD6Q\n/xXtR268ThL16SR2k8wzq5bEbQwfl7CSb7T443f8mRkjYzMzrHTjHONv/aEkadiK4A0xds6FzTIl\n6ZJlV0qS2u4/OeHY+AzFfs7fnp72n+AhEgUABzSiAOCgZAaW7KBEqgGJhrca2z23lwompYGS1Hrg\nQBZXmeLc/WolSdEvDkqS2lpacjoPAxCZs+s92np0rUPvvLG6lNzr06rkes1kamK85GmKt8/6Xeix\nYdPcJOlbt9yc8Hs2g4yppPq7WHvofxhYAoBCK5lItKp7d3NBg0+RlDQgMeH/CnthRVLJEUs5q+R6\ntTdYhN1ckcxOU7SDxJmw09wkadOo/K0D2h4m2wNAEZTMFCevX+rd2C2VEzruWgBkZtIgs55npjda\n3PrejHbPmTy+UerrvRKJAoADGlEAcFAy6TyAzivt3WoJA8PtDxJnNkRVOohEAcABkSgAZwk3LFTY\n4DCRKAA4oBEFAAc0ogDggEYUABzQiAKAAxpRAHBAIwoADmhEAcBBUdcTBYByQyQKAA5oRAHAAY0o\nADigEQUABzSiAOCARhQAHNCIAoADGlEAcEAjCgAOaEQBwAGNKAA4oBEFAAdFfdrnJVUzKnq1k3Vt\nKyMdfQ2FQL1Sr+Uo03olEgUABxXx3PnqPn0kSZEeNe0ee3zXx4W+HABlhEgUABzQiAKAg4pI57Ox\n7/pxkqTaqz8I7IvMNd0Cba9tK+o1AShdRKIA4KAiItFtC4dLkt5ueMjpPJNPvFES3zwAfLQHAOCg\nbCPRlobzvPLiCUvycs7xi16UJO06coK3bfvcEZKkqr9syct7wMhmWtruycMkSX0febGg14TCsfUt\nBeu81KcdEokCgIOyjUSn373WK19c05KXc/60dqsk6amDJ3nbtnU7SxLfRvmWTT/2BbfPkSRF/nRq\nYB8zKjoHW99S+jq3dZ08e8bWs1T8uuazDwAOaEQBwEHZpvMr7rjMK49auFiSNK5ba+C4UQ/eIkka\nsvbzhO3NU3t55fVNCyVJA6pNh/fUnvu9ffO+bf4L6zeYn9Hjx52vvVLlOhj4wi/uC93HtLTSZus8\n0/oOq2tbz1Lx65q/LQBwULaRaFWrvxTirOWzQ48b+uxnkqToq1sTt//VL/9m6vmSpPn9Xg+8fvu0\nByRJjT83kW/rnj25XTAKMhiYPC3NTkmTmJZWClq7mSU7Xet7+D3/65W3zx0tqXj1SyQKAA7KNhKt\nWf2yV65bHX5cJkt3b/jZBZKk+Q8HI1HkT6b92Nmw09Kspx7xp788MHuGJKnL+s1O74Hc9Vz1kiRp\n5Kg53rY3bkjs97TjFlL42MWb18e9ZsULkqTLrp0lqfD1SyQKAA5oRAHAQdmm8/nUbf+Rjr6EimBT\nO0nSwvaPD5ueJvlpHtPTOoe6O/x1DxruGJOwb8g5n3nl0AHg64Pn/MOy/5IkTak73399AeqYSBQA\nHBCJZmDX2F7tH4S8mn+rGRTY/bXq0GPCpqdJfoTC9LTOL1X9ZqPqJH+ti0LUMZEoADggEs3AtJkb\nOvoSKo6dosb0NKQ19uxYoeOmqRGJAoCDiopEW6aYxQ72Dff/2VWxudwDf/1CwrHRC0d75XN6PBZ6\nzjk7LzKFI4zglyJmVpS3sYs7/kYJIlEAcEAjCgAOOmU6X32i/6C4SF8zfWHHdwdJkmr2mOGG+plv\nBV533QCzZmH8ijHHoiafv3H6pIRjL61d45Uv7/FZwr5799d75Q+vOkWS1Hrg/Sz/FSikXf9sBpT+\nNm9R6DGNo5naVAypPq8ff8t8blJ9Tnf++z9J8rvdquKWT0judjunx9/zeq25IBIFAAedIxKNTWPY\n0dBTktT/XP8Rqs+PXOl06q4RM5n7t0OfS9j+u89PcTovOhbT0jqQ6+f1ocTPos0WJT9jvDS2Oldy\nlij5meLzV4wyG47szfDCc0MkCgAOOkUk2jzVfKNtbQp/lo61t/WwJKlfbMGJXF3V+x+h+07rus8r\nv9c0QJI0bMEuSVLboUNO74tEydPS0vWPZTItzZuSJjEtrUDWrFoqSWpLczuE/Zyu+Nw8cnxQV39h\nmCt67ks41maLUjBjTMV+PpM/m4VCJAoADmhEAcBBp0jnc3H/p2d4ZZsqJKcJubqyl99RfeVM08Uw\nesS1kqS6m3ZLYtpMMjvNxU5xkXKblpZqkMHKZFqanZImMS2tUKojJjZriwYf7dK0Y6Ik6Y0nzAMD\nB91jumSqz/TX/Ny8fLsk6a6Tc7sbyX4+kz+bUmE+n0SiAOCgU0Si25rulyS1pTkm7BtO8r/lMvmG\naz5uIp7Lf//jwL7zv2Eecrakbn1gX20vM6BEBBoTMs0lX1PSpMwGGazkwQaJwcBCGbHxGknS6xcs\nDezz6myu+fnErL6xPc3eMWO67YyVgoPD42/9oSSp94pNkqT37x4nyf9sSsHP55axy7zyhOXTzZkn\nEYkCQEmIRKOZrMqYH5dUzcjpzdZ+tEVSYn9YsrePHZUkbT06MPQY+w03pEvwG25jS1dJ0i/PODuw\nr8tAE70cXGZed/sZT3v7vtn9aMKxDacmPh8m3rq2lZHQnZ1YqnptXmAihGympdnpLlL++7FTGb0p\nP/3YlVSvmajq3l2SFBkcuyV68bF2X/PMV54MbLtrr/ksPrZivLdt8IIXAsclO/LHoZL8z2n8ZzTd\n5zNZpvVKJAoADjpFJPrBypGSUvexZCPVN1vft0x02+Pxl1K+JpXDjed55UcX/UqSNHHTbElS3Xfe\nCH1dJUUsa3a+Iin9hOv0/djDJUkjC9iPbU140/aTNYcek04l1WsxVPfvbwqxmyFaDxzI6Tz2c2o/\no5J0w5CLwg4PIBIFgCKgEQUAB50inc+lo7p6Ttzg0d5PzU/H9CDl+/SrlSRFvzgoSWpraQk9tpLS\nvmIOBt52m3m8sp32Ei9fg4LpVFK9VhLSeQAogk4x2d6L7t6NdfxPaP814fFPfrXu/aRI79S5pJtw\nbdV3/VLsZ7ppTCaCTDco2PvxYARqHd9lJvl3u9T8fmfjDd6+wKCgwgcFgTBEogDgoFNEouh8hl7z\njiSpcbCZPpRJP3Y8r087qT978IH2J1unU/Pky1551sZGSdLpX5hrTXdbMRCGSBQAHNCIAoAD0nkU\nRC6DgfGKMTDIoCDygUgUABzQiAKAAxpRAHBAIwoADmhEAcABjSgAOKARBQAHNKIA4IBGFAAc0IgC\ngIOirmwPAOWGSBQAHNCIAoADGlEAcEAjCgAOaEQBwAGNKAA4oBEFAAc0ogDggEYUABzQiAKAAxpR\nAHBAIwoADmhEAcABjSgAOKARBQAHNKIA4IBGFAAc0IgCgAMaUQBwQCMKAA5oRAHAAY0oADigEQUA\nB/8PvsM/W/0NKNgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6ca03a7ba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-fef3b07aa51d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTTA_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "### debugging section\n",
    "x = test_x[0,:]\n",
    "datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.1,\n",
    "        zoom_range=0.2, fill_mode='nearest')\n",
    "x2 = x.reshape((1,)+x.shape)\n",
    "\n",
    "TTA_batch = np.zeros((9,)+test_x.shape[1:])\n",
    "i = -1\n",
    "for batch in datagen.flow(x2,batch_size=1):\n",
    "    i+=1\n",
    "    if i>8:\n",
    "        break # otherwise the generator would loop indefinitely\n",
    "    TTA_batch[i,:] = batch.reshape(x.shape)\n",
    "    #plt.imshow(TTA_batch[i,:].reshape(x.shape)[:,:,0])\n",
    "    #plt.show()\n",
    "\n",
    "print(TTA_batch.shape)\n",
    "i = 0\n",
    "for im in TTA_batch:\n",
    "    i+=1\n",
    "    temp = im.reshape(x.shape)\n",
    "    plt.subplot(3,3,i)\n",
    "    #print(temp.shape)\n",
    "    if i == 0:\n",
    "        plt.imshow(x[:,:,0])\n",
    "    else:\n",
    "        plt.imshow(temp[:,:,0])\n",
    "    plt.axis('off')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "preds = model.predict(TTA_batch)\n",
    "print(preds.shape)\n",
    "preds = preds.mean(axis = 0)\n",
    "print(preds.shape)\n",
    "predss = np.zeros((2,10))\n",
    "predss[0,:] = preds\n",
    "predss[1,:] = preds\n",
    "print(predss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}