7.PRETRAIN_METHOD/7.2.5.bert_finetune_KorQuAD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12607,
     "status": "ok",
     "timestamp": 1594010753269,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "B9WLyWEWgdDR"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import json\n",
    "import string\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tokenizers import BertWordPieceTokenizer\n",
    "from transformers import BertTokenizer, TFBertModel\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "import matplotlib.pyplot as plt\n",
    "import urllib\n",
    "import wget\n",
    "\n",
    "MAX_LEN = 384\n",
    "EPOCHS = 3\n",
    "VERBOSE = 2\n",
    "BATCH_SIZE = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 556,
     "status": "ok",
     "timestamp": 1594010762115,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "68HVB3dYgi0w"
   },
   "outputs": [],
   "source": [
    "DATA_OUT_PATH = './data_out/KOR'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 639,
     "status": "ok",
     "timestamp": 1594010763471,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "zvoswBdyglTQ"
   },
   "outputs": [],
   "source": [
    "def plot_graphs(history, string, string_1, string_2):\n",
    "    # loss \n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history[string_1])\n",
    "    plt.plot(history.history[string_2])\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, string_1, string_2])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED_NUM = 1234\n",
    "tf.random.set_seed(SEED_NUM)\n",
    "np.random.seed(SEED_NUM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 65,
     "referenced_widgets": [
      "bc7f3c579a324f77811bdd6ad6dd7dc0",
      "e31de13423d743e68d6c451d23c93cdf",
      "f8f80478dfca4894ac1ff8c2a082f734",
      "3be3c9704e934fb5a3d5847749d398ce",
      "2c0ecef646d44a0580cacefa5c3fd9f2",
      "1fde406732df4b5b90b7701dc7e4981e",
      "f58154a65f974e04bcf8af24b2884fdd",
      "a7d4d0c48cda4abdb106a6bcfb24359e"
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1217,
     "status": "ok",
     "timestamp": 1594010812799,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "HDI_cm3sgm6N",
    "outputId": "33078a97-0007-428b-9439-b67bd53cd994"
   },
   "outputs": [],
   "source": [
    "# Save the slow pretrained tokenizer\n",
    "slow_tokenizer = BertTokenizer.from_pretrained(\"bert-base-multilingual-cased\", lowercase=False)\n",
    "save_path = \"bert-base-multilingual-cased/\"\n",
    "if not os.path.exists(save_path):\n",
    "    os.makedirs(save_path)\n",
    "slow_tokenizer.save_pretrained(save_path)\n",
    "\n",
    "# Load the fast tokenizer from saved file\n",
    "tokenizer = BertWordPieceTokenizer(\"bert-base-multilingual-cased/vocab.txt\", lowercase=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 83
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1750,
     "status": "ok",
     "timestamp": 1594010820826,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "an5cGi-GgpG4",
    "outputId": "c7753a24-f338-4a6d-8701-f78753f9b718"
   },
   "outputs": [],
   "source": [
    "train_data_url = \"https://korquad.github.io/dataset/KorQuAD_v1.0_train.json\"\n",
    "train_path = keras.utils.get_file(\"train.json\", train_data_url)\n",
    "eval_data_url = \"https://korquad.github.io/dataset/KorQuAD_v1.0_dev.json\"\n",
    "eval_path = keras.utils.get_file(\"eval.json\", eval_data_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./bert-base-multilingual-cased//bert-base-multilingual-cased-config.json'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wget.download('https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-config.json', out='./bert-base-multilingual-cased/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.rename('./bert-base-multilingual-cased/bert-base-multilingual-cased-config.json', './bert-base-multilingual-cased/config.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./bert-base-multilingual-cased//bert-base-multilingual-cased-tf_model.h5'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wget.download('https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-tf_model.h5', out='./bert-base-multilingual-cased/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.rename('./bert-base-multilingual-cased/bert-base-multilingual-cased-tf_model.h5', './bert-base-multilingual-cased/tf_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 99893,
     "status": "ok",
     "timestamp": 1594011009085,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "PkuK7N_ngrMd",
    "outputId": "48275df3-52de-4623-dfc3-db6be9a54dfa"
   },
   "outputs": [],
   "source": [
    "class SquadExample:\n",
    "    def __init__(self, question, context, start_char_idx, answer_text):\n",
    "        self.question = question\n",
    "        self.context = context\n",
    "        self.start_char_idx = start_char_idx\n",
    "        self.answer_text = answer_text\n",
    "        self.skip = False\n",
    "\n",
    "    def preprocess(self):\n",
    "        context = self.context\n",
    "        question = self.question\n",
    "        answer_text = self.answer_text\n",
    "        start_char_idx = self.start_char_idx\n",
    "\n",
    "        # Clean context, answer and question\n",
    "        context = \" \".join(str(context).split())\n",
    "        question = \" \".join(str(question).split())\n",
    "        answer = \" \".join(str(answer_text).split())\n",
    "\n",
    "        # Find end character index of answer in context\n",
    "        end_char_idx = start_char_idx + len(answer)\n",
    "        if end_char_idx >= len(context):\n",
    "            self.skip = True\n",
    "            return\n",
    "\n",
    "        # Mark the character indexes in context that are in answer\n",
    "        is_char_in_ans = [0] * len(context)\n",
    "        for idx in range(start_char_idx, end_char_idx):\n",
    "            is_char_in_ans[idx] = 1\n",
    "\n",
    "        # Tokenize context\n",
    "        tokenized_context = tokenizer.encode(context)\n",
    "\n",
    "        # Find tokens that were created from answer characters\n",
    "        ans_token_idx = []\n",
    "        for idx, (start, end) in enumerate(tokenized_context.offsets):\n",
    "            if sum(is_char_in_ans[start:end]) > 0:\n",
    "                ans_token_idx.append(idx)\n",
    "\n",
    "        if len(ans_token_idx) == 0:\n",
    "            self.skip = True\n",
    "            return\n",
    "\n",
    "        # Find start and end token index for tokens from answer\n",
    "        start_token_idx = ans_token_idx[0]\n",
    "        end_token_idx = ans_token_idx[-1]\n",
    "\n",
    "        # Tokenize question\n",
    "        tokenized_question = tokenizer.encode(question)\n",
    "\n",
    "        # Create inputs\n",
    "        input_ids = tokenized_context.ids + tokenized_question.ids[1:]\n",
    "        token_type_ids = [0] * len(tokenized_context.ids) + [1] * len(\n",
    "            tokenized_question.ids[1:]\n",
    "        )\n",
    "        attention_mask = [1] * len(input_ids)\n",
    "\n",
    "        # Pad and create attention masks.\n",
    "        # Skip if truncation is needed\n",
    "        padding_length = MAX_LEN - len(input_ids)\n",
    "        if padding_length > 0:  # pad\n",
    "            input_ids = input_ids + ([0] * padding_length)\n",
    "            attention_mask = attention_mask + ([0] * padding_length)\n",
    "            token_type_ids = token_type_ids + ([0] * padding_length)\n",
    "        elif padding_length < 0:  # skip\n",
    "            self.skip = True\n",
    "            return\n",
    "\n",
    "        self.input_ids = input_ids\n",
    "        self.token_type_ids = token_type_ids\n",
    "        self.attention_mask = attention_mask\n",
    "        self.start_token_idx = start_token_idx\n",
    "        self.end_token_idx = end_token_idx\n",
    "        self.context_token_to_char = tokenized_context.offsets\n",
    "\n",
    "\n",
    "def create_squad_examples(raw_data):\n",
    "    squad_examples = []\n",
    "    for item in raw_data[\"data\"]:\n",
    "        for para in item[\"paragraphs\"]:\n",
    "            context = para[\"context\"]\n",
    "            for qa in para[\"qas\"]:\n",
    "                question = qa[\"question\"]\n",
    "                answer_text = qa[\"answers\"][0][\"text\"]\n",
    "                start_char_idx = qa[\"answers\"][0][\"answer_start\"]\n",
    "                squad_eg = SquadExample(\n",
    "                    question, context, start_char_idx, answer_text\n",
    "                )\n",
    "                squad_eg.preprocess()\n",
    "                squad_examples.append(squad_eg)\n",
    "    return squad_examples\n",
    "\n",
    "\n",
    "def create_inputs_targets(squad_examples):\n",
    "    dataset_dict = {\n",
    "        \"input_ids\": [],\n",
    "        \"token_type_ids\": [],\n",
    "        \"attention_mask\": [],\n",
    "        \"start_token_idx\": [],\n",
    "        \"end_token_idx\": [],\n",
    "    }\n",
    "    for item in squad_examples:\n",
    "        if item.skip == False:\n",
    "            for key in dataset_dict:\n",
    "                dataset_dict[key].append(getattr(item, key))\n",
    "    for key in dataset_dict:\n",
    "        dataset_dict[key] = np.array(dataset_dict[key])\n",
    "\n",
    "    x = [\n",
    "        dataset_dict[\"input_ids\"],\n",
    "        dataset_dict[\"token_type_ids\"],\n",
    "        dataset_dict[\"attention_mask\"],\n",
    "    ]\n",
    "    y = [dataset_dict[\"start_token_idx\"], dataset_dict[\"end_token_idx\"]]\n",
    "    return x, y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60407 training points created.\n",
      "5774 evaluation points created.\n"
     ]
    }
   ],
   "source": [
    "with open(train_path) as f:\n",
    "    raw_train_data = json.load(f)\n",
    "\n",
    "with open(eval_path) as f:\n",
    "    raw_eval_data = json.load(f)\n",
    "\n",
    "\n",
    "train_squad_examples = create_squad_examples(raw_train_data)\n",
    "x_train, y_train = create_inputs_targets(train_squad_examples)\n",
    "print(f\"{len(train_squad_examples)} training points created.\")\n",
    "\n",
    "eval_squad_examples = create_squad_examples(raw_eval_data)\n",
    "x_eval, y_eval = create_inputs_targets(eval_squad_examples)\n",
    "print(f\"{len(eval_squad_examples)} evaluation points created.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 690,
     "status": "ok",
     "timestamp": 1594011009787,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "mIjk3_XeguBj"
   },
   "outputs": [],
   "source": [
    "class TFBERTQuestionAnswering(tf.keras.Model):\n",
    "    def __init__(self, model_name, dir_path, num_class):\n",
    "        super(TFBERTQuestionAnswering, self).__init__()\n",
    "        \n",
    "        self.encoder = TFBertModel.from_pretrained(model_name, cache_dir=dir_path)\n",
    "        self.start_logit = tf.keras.layers.Dense(num_class, name=\"start_logit\", use_bias=False)\n",
    "        self.end_logit = tf.keras.layers.Dense(num_class, name=\"end_logit\", use_bias=False)\n",
    "        self.flatten = tf.keras.layers.Flatten() \n",
    "        self.softmax = tf.keras.layers.Activation(tf.keras.activations.softmax)\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        input_ids, token_type_ids, attention_mask = inputs\n",
    "        embedding = self.encoder(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)[0]\n",
    "        start_logits = self.start_logit(embedding)\n",
    "        start_logits = self.flatten(start_logits)\n",
    "        \n",
    "        end_logits = self.end_logit(embedding)\n",
    "        end_logits = self.flatten(end_logits)\n",
    "        \n",
    "        start_probs = self.softmax(start_logits)\n",
    "        end_probs = self.softmax(end_logits)\n",
    "    \n",
    "        return start_probs, end_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 120
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11135,
     "status": "ok",
     "timestamp": 1594011020239,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "k4t_2T7vgwOu",
    "outputId": "fd7dcb5d-bf36-496c-b53d-53e89962360a"
   },
   "outputs": [],
   "source": [
    "korquad_model = TFBERTQuestionAnswering(model_name='./bert-base-multilingual-cased/',dir_path='bert_ckpt', num_class=1)\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)\n",
    "loss = keras.losses.SparseCategoricalCrossentropy(from_logits=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 590,
     "status": "ok",
     "timestamp": 1594011103474,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "YZtVFA3PgyL0"
   },
   "outputs": [],
   "source": [
    "def normalized_answer(s):    \n",
    "    def remove_(text):\n",
    "        ''' 불필요한 기호 제거 '''\n",
    "        text = re.sub(\"'\", \" \", text)\n",
    "        text = re.sub('\"', \" \", text)\n",
    "        text = re.sub('《', \" \", text)\n",
    "        text = re.sub('》', \" \", text)\n",
    "        text = re.sub('<', \" \", text)\n",
    "        text = re.sub('>', \" \", text) \n",
    "        text = re.sub('〈', \" \", text)\n",
    "        text = re.sub('〉', \" \", text)   \n",
    "        text = re.sub(\"\\(\", \" \", text)\n",
    "        text = re.sub(\"\\)\", \" \", text)\n",
    "        text = re.sub(\"‘\", \" \", text)\n",
    "        text = re.sub(\"’\", \" \", text)      \n",
    "        return text\n",
    "\n",
    "    def white_space_fix(text):\n",
    "        return ' '.join(text.split())\n",
    "\n",
    "    def remove_punc(text):\n",
    "        exclude = set(string.punctuation)\n",
    "        return ''.join(ch for ch in text if ch not in exclude)\n",
    "\n",
    "    def lower(text):\n",
    "        return text.lower()\n",
    "\n",
    "    return white_space_fix(remove_punc(lower(remove_(s))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 720,
     "status": "ok",
     "timestamp": 1594011104061,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "rVTh1qKng1p8"
   },
   "outputs": [],
   "source": [
    "class ExactMatch(keras.callbacks.Callback):\n",
    "    def __init__(self, x_eval, y_eval):\n",
    "        self.x_eval = x_eval\n",
    "        self.y_eval = y_eval\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        pred_start, pred_end = self.model.predict(self.x_eval)\n",
    "        count = 0\n",
    "        eval_examples_no_skip = [_ for _ in eval_squad_examples if _.skip == False]\n",
    "        for idx, (start, end) in enumerate(zip(pred_start, pred_end)):\n",
    "            squad_eg = eval_examples_no_skip[idx]\n",
    "            offsets = squad_eg.context_token_to_char\n",
    "            start = np.argmax(start)\n",
    "            end = np.argmax(end)\n",
    "            if start >= len(offsets):\n",
    "                continue\n",
    "            pred_char_start = offsets[start][0]\n",
    "            if end < len(offsets):\n",
    "                pred_char_end = offsets[end][1]\n",
    "                pred_ans = squad_eg.context[pred_char_start:pred_char_end]\n",
    "            else:\n",
    "                pred_ans = squad_eg.context[pred_char_start:]\n",
    "\n",
    "            normalized_pred_ans = normalized_answer(pred_ans)\n",
    "            normalized_true_ans = normalized_answer(squad_eg.answer_text)\n",
    "            if normalized_pred_ans in normalized_true_ans:\n",
    "                count += 1\n",
    "        acc = count / len(self.y_eval[0])\n",
    "        print(f\"\\nepoch={epoch+1}, exact match score={acc:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 399,
     "status": "ok",
     "timestamp": 1594011104303,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "sTgvtk0og4Ow"
   },
   "outputs": [],
   "source": [
    "exact_match_callback = ExactMatch(x_eval, y_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 599,
     "status": "ok",
     "timestamp": 1594011105561,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "7EuBYS58g6QZ"
   },
   "outputs": [],
   "source": [
    "korquad_model.compile(optimizer=optimizer, loss=[loss, loss])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 714,
     "status": "ok",
     "timestamp": 1594011106252,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "ZehxFPSrg8Q2",
    "outputId": "6a33f8a1-84d0-48c4-ac1e-5843daf1f2fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data_out/KOR/tf2_bert_korquad -- Folder create complete \n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_name = \"tf2_bert_korquad\"\n",
    "\n",
    "checkpoint_path = os.path.join(DATA_OUT_PATH, model_name, 'weights.h5')\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "# Create path if exists\n",
    "if os.path.exists(checkpoint_dir):\n",
    "    print(\"{} -- Folder already exists \\n\".format(checkpoint_dir))\n",
    "else:\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    print(\"{} -- Folder create complete \\n\".format(checkpoint_dir))\n",
    "    \n",
    "cp_callback = ModelCheckpoint(\n",
    "    checkpoint_path, verbose=1, save_best_only=True, save_weights_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 383
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 18126376,
     "status": "ok",
     "timestamp": 1594029233934,
     "user": {
      "displayName": "ChangWook Jun",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjljUh9LMLCM8kMgWLaX2xHiw2Cej8KoaOlkKxE=s64",
      "userId": "00685987924881157185"
     },
     "user_tz": -540
    },
    "id": "2ljuajCLmyws",
    "outputId": "e89526e8-e795-48df-eead-1a00b28005bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "\n",
      "epoch=1, exact match score=0.72\n",
      "WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "2941/2941 - 1381s - loss: 1.6155 - output_1_loss: 0.7417 - output_2_loss: 0.8737\n",
      "Epoch 2/3\n",
      "\n",
      "epoch=2, exact match score=0.74\n",
      "WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "2941/2941 - 1378s - loss: 1.0490 - output_1_loss: 0.4733 - output_2_loss: 0.5756\n",
      "Epoch 3/3\n",
      "\n",
      "epoch=3, exact match score=0.74\n",
      "WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "2941/2941 - 1379s - loss: 0.8303 - output_1_loss: 0.3712 - output_2_loss: 0.4590\n"
     ]
    }
   ],
   "source": [
    "history = korquad_model.fit(\n",
    "    x_train,\n",
    "    y_train,\n",
    "    epochs=EPOCHS,  # For demonstration, 3 epochs are recommended\n",
    "    verbose=VERBOSE,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    callbacks=[exact_match_callback, cp_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': [1.6154567003250122, 1.0489535331726074, 0.8302596807479858], 'output_1_loss': [0.7417199611663818, 0.473328560590744, 0.3712247312068939], 'output_2_loss': [0.8737384080886841, 0.575624406337738, 0.45903465151786804]}\n"
     ]
    }
   ],
   "source": [
    "print(history.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QxaigHy2m4JB"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU5bnA8d+bfd/IvrPvEPZFCCRoRaFFq9WqteJSr+11r1Z7bdVKb2sr1qVavdai0taldSuuKCbsixBk32TLDgnZF7K/949zMpmEJASYyWQyz/fzmQ8zZ87MPDkMPHm351Vaa4QQQrguN0cHIIQQwrEkEQghhIuTRCCEEC5OEoEQQrg4SQRCCOHiPBwdwLkKDw/XycnJjg5DCCGcSlZW1imtdURnzzldIkhOTmbbtm2ODkMIIZyKUiq7q+eka0gIIVycJAIhhHBxkgiEEMLFOd0YgRDC8RobG8nLy6Ours7RoYgOfHx8iI+Px9PTs8evkUQghDhneXl5BAYGkpycjFLK0eEIk9aakpIS8vLyGDhwYI9fJ11DQohzVldXx4ABAyQJ9DFKKQYMGHDOLTVJBEKI8yJJoG86n78Xl0kEJdX1LPl4H+W1DY4ORQgh+hS7JQKl1DKlVJFSak8358xVSu1QSu1VSq2xVywAG46U8NqGY6QtXc0/t2TT3CL7MAjhrAICAhwdQr9izxbB68D8rp5USoUAfwG+p7UeDfzAjrHwvfGxfHL3bIZGBfLIB3tY9OJ6srLL7PmRQgjhFOyWCLTWa4HSbk65Hnhfa51jnl9kr1hajYwJ4p3bp/PcD1Morqrnqpc28vN/7aSoSqbACeGMtNY8+OCDjBkzhrFjx/LOO+8AUFhYSGpqKikpKYwZM4Z169bR3NzM4sWLLec+88wzDo6+73Dk9NFhgKdSajUQCDyntV7e2YlKqduB2wESExMv6EOVUixKiePikVH8OeMwf1t/lJV7T3DvxUO5aWYynu4uM2wihE385qO97CuotOl7jooN4rHvjj7ree+//z47duxg586dnDp1iilTppCamsqbb77JpZdeyiOPPEJzczO1tbXs2LGD/Px89uwxeqvLy8ttGrMzc+T/eh7AJGABcCnwa6XUsM5O1Fq/orWerLWeHBHRafG8c+bv7cHDl41g5b2pTEoK5bef7Oey59ax4fApm7y/EML+1q9fz3XXXYe7uztRUVHMmTOHrVu3MmXKFF577TUef/xxdu/eTWBgIIMGDeLo0aPcddddfP755wQFBTk6/D7DkS2CPOCU1roGqFFKrQXGA4d6M4hBEQG8fvMUVu0v4omP93LDq1u4bEw0jywYSXyoX2+GIoRT6slv7vaideeTPlJTU1m7di2ffPIJN954Iw8++CA//vGP2blzJytXruTFF1/kX//6F8uWLevliPsmR7YI/gPMVkp5KKX8gGnAfkcEopTiklFRfHnfHO6/ZBiZB4u4+E9r+PNX31LX2OyIkIQQPZCamso777xDc3MzxcXFrF27lqlTp5KdnU1kZCQ/+clPuPXWW9m+fTunTp2ipaWFq666iiVLlrB9+3ZHh99n2K1FoJR6C5gLhCul8oDHAE8ArfXLWuv9SqnPgV1AC/Cq1rrLqaa9wcfTnbvnDeX7E+P43af7efrLQ/w7K49fLxzFxSMjZQGNEH3MlVdeyaZNmxg/fjxKKf74xz8SHR3NG2+8wVNPPYWnpycBAQEsX76c/Px8br75ZlpaWgD4/e9/7+Do+w7VVdOqr5o8ebLurY1pNhw+xWMr9nK4qJq5wyN4dOEoBkXI/GUh9u/fz8iRIx0dhuhCZ38/SqksrfXkzs6XKTLduGhIOJ/dM5tfLRjJtuNlXPrsWp787AA19U2ODk0IIWxGEsFZeLq7cdvsQWQ8MIfvjo/l5TVHmPf0Gv6zI7/LgSohhHAmkgh6KDLQhz9dk8J7P51BeKAX97y9g2tf2cz+QtvOnxZCiN4mieAcTUoK4z//PYv/vXIMh05WseD5dTz2nz1U1DY6OjQhhDgvkgjOg7ub4oZpSax+YC43TEvi75uzSXt6NW9/nUOLFLMTQjgZSQQXIMTPiyVXjOGju2YxKNyfh9/fzZV/2cCOXFm6LoRwHpIIbGB0bDD/vmMGz16bQmFFHVe8uIFfvLuTU9X1jg5NCCHOShKBjSiluGJCHBkPzOW/Ugfx/vZ80pauZtn6YzQ1tzg6PCFc2uuvv05BQcF5v/748eO8+eab3Z5TUlJCWloaAQEB3HnnnWd9z+TkZE6d6hu1zSQR2FiAtwe/vHwkn9+bSkpCCE98vI8Fz69n05ESR4cmhMvqjUTg4+PDkiVLWLp06Xl/jqM4suhcvzYkMoDlt0xl5d6TLPl4H9f9dTMLxsXwyOUjiQ3xdXR4QtjOZw/Did22fc/osXDZk92e8qc//clSNO62227jiiuuYOHChZYy00uXLqW6upoxY8awbds2brjhBnx9fdm0aRMjR47k2muvJTMzE4A333yTIUOGsHjxYhYuXMjVV18NGDuhVVdX8/DDD7N//35SUlK46aabuO+++86Ix9/fn1mzZnH48OFz/nE7/iz33nsvNTU1XHPNNeTl5dHc3Myvf/1rrr32Wh5++GFWrFiBh4cH3/nOd2ySeCQR2JFSivljopk7PIKXVh/h5TVHyNhfxJ3pQ7ht9kC8PdwdHaIQTikrK4vXXnuNLVu2oLVm2rRpzJkzp9Nzr776al544QWWLl3K5MltFRaCgoL4+uuvWb58Offeey8ff/xxl5/35JNPsnTp0m7PsfXPcvToUWJjY/nkk08AqKiooLS0lA8++IADBw6glLLZngqSCHqBj6c7910yjKsnxfPbT/bx1MqD/HtbLo9+dxTpI6IcHZ4QF+Ysv7nbw/r167nyyivx9/cH4Pvf/z7r1q07p/e47rrrLH929ht+b+nqZ5k/fz4PPPAADz30EAsXLmT27Nk0NTXh4+PDbbfdxoIFC1i4cKFNYpAxgl6UEObH/904meW3TMXNTXHL69u49fWtHD9V4+jQhHAqnZV3KS8vt1QWBair634LWutqwq33PTw8LO+htaahocEW4Xarq1I1w4YNIysri7Fjx/LLX/6SJ554Ag8PD77++muuuuoqPvzwQ+bP73Jb+HMiicABUodF8Pk9qfzP5SPYfLSE7zyzlqdWHqC2QYrZCdETqampfPjhh9TW1lJTU8MHH3zAZZddRlFRESUlJdTX17frxgkMDKSqqqrde7Tub/zOO+8wY8YMwJjJk5WVBcB//vMfGhsbu3y9PX+W2bNnU1BQgJ+fHz/60Y944IEH2L59O9XV1VRUVHD55Zfz7LPPsmPHDpvEIF1DDuLl4cbtqYNZlBLHk58d4MXMI7y/PZ9HFoxkwdgY2ftAiG5MnDiRxYsXM3XqVMAYYJ0yZQqPPvoo06ZNY+DAgYwYMcJy/uLFi7njjjssg8UA9fX1TJs2jZaWFt566y0AfvKTn7Bo0SKmTp3KvHnzLN0148aNw8PDg/Hjx7N48eIuu5KSk5OprKykoaGBDz/8kC+++IJRo0ad888yYcIEVq5cyYMPPoibmxuenp689NJLVFVVsWjRIurq6tBa88wzz1zYhTTJfgR9xNbjpTz6n73sL6xkxqABPP690QyPDnR0WEJ0ytn3I0hOTmbbtm2Eh4c7OhS7kP0InNSU5DA+vmsWSxaNZl9hJZc/v47ffLSXitNSzE4IYV/SNdSHuLspbpyRzIJxsSz94iCvbzzORzsL+MX8EVw9MR43N+kuEsIWjh8/ft6vXblyJQ899FC7YwMHDuSDDz7o9Pxp06ZRX9++3Mzf//53xo4de94x2Jp0DfVhu/MqeGzFHrbnlJOSEMJvvjea8Qkhjg5LCKfvGurvpGuoHxkbH8y7d8zk6R+MJ6/sNFf8ZQMPv7eLEilmJ4SwIUkEfZybm+KqSfFkPDCHWy8ayLtZeaQtXc0bG49LMTshhE1IInASQT6e/GrhKD67ZzZj44N5bMVeFv55PVuOSjE7IcSFsVsiUEotU0oVKaX2nOW8KUqpZqXU1faKpT8ZGhXIP26dxks3TKSqrolrX9nM3W99w4mK7ldRCiFEV+zZIngd6Hb9s1LKHfgDsNKOcfQ7SikuGxvDqvvncHf6ED7fe4L0p1fz0uoj1Dc1Ozo8Ifqc3ihD/eWXXzJp0iTGjh3LpEmTyMjI6PZ8l9iPQGu9Fig9y2l3Ae8BRfaKoz/z9XLn/u8MZ9V9c5g5OJw/fH6A+c+uY/VBuZxCWOuNRBAeHs5HH33E7t27eeONN7jxxhvP+/N6m8PWESil4oArgXRgylnOvR24HSAxMdH+wTmZxAF+vHrTZDIPFvHER/tY/NpWLh4ZxaMLR5E4wM/R4Yl+7g9f/4EDpQds+p4jwkbw0NSHuj2nr+1HMGHCBMv90aNHU1dXR319Pd7e3mf9eV15P4JngYe01s1nq6ujtX4FeAWMdQS9EJtTShseyczBA1i2/jh/zviWi59Zwx2pg/jp3CH4esneB6L/6Ov7Ebz33ntMmDChR0nA1fcjmAy8bSaBcOBypVST1vpDB8bk9Lw93Pnp3MFcOSGO3326n+czDvPe9nx+tWAk88dESzE7YXNn+83dHvryfgR79+7loYce4osvvujR+S69H4HWeqDWOllrnQy8C/xMkoDtRAf78Px1E3j79ukE+njw039u50d/28K3J+1TSleI3tRX9yPIy8vjyiuvZPny5QwePLhHr+nX+xEopd4CNgHDlVJ5SqlblVJ3KKXusNdnijNNHzSAj++axW++N5rdeRVc9tw6fvvxPqrqpJidcF59cT+C8vJyFixYwO9//3suuuiiC/pZ+s1+BFrr687h3MX2ikOAh7sbN81MZuG4GJ5aeZC/bTjGhzsK+OVlI7hyQpwUsxNOpy/uR/DCCy9w+PBhlixZwpIlSwD44osviIyMPOefRfYjOAtXKjpnLztzy3l0xV525pYzMTGEJxaNYUxcsKPDEk7E2YvOyX4E7UmJCRc0PiGED346kz9ePY7sklq++8J6/ueD3ZTV2H9/ViFE3yP7EbgoNzfFNZMTuHR0NM+uOsTyTdl8squQBy4dzvVTE3GX7iLRj8l+BO1J15AA4OCJKh5bsYfNR0sZFRPEE4tGMzk5zNFhiT5q//79jBgxQqYj90Faaw4cOCBdQ+LcDY8O5K2fTOeF6ydQVtvA1S9v4r53dlBUKcXsxJl8fHwoKSnpcuqjcAytNSUlJfj4+JzT66RFIM5Q29DEi5mH+evaY3i6K+65eCiLZw7Ey0N+bxCGxsZG8vLyzjpXX/Q+Hx8f4uPj8fT0bHe8uxaBJALRpeOnanji431kHChiUIQ/j393NKnDIhwdlhDiPEjXkDgvyeH+LFs8hb/dNJnmFs2Pl33Nf/19G7mltY4OTQhhQ5IIxFnNGxnFyntTefDS4aw9dIqL/7SGZ1cdoq5R9j4Qoj+QRCB6xMfTnf9OG8JXP5/DxaOieHbVt1z8pzV8vueEDBgK4eQkEYhzEhviy4vXT+TNn0zDz8udO/6RxY+Xfc2R4mpHhyaEOE+SCMR5mTk4nE/uns2jC0exI6ec+c+u5fef7qe6vsnRoQkhzpEkAnHePN3duGXWQDIemMsVKXH839qjpC9dzYff5Et3kRBORBKBuGARgd489YPxfPCzmUQH+3DvOzu45v82sbegwtGhCSF6QBKBsJkJiaF8+LOLePL7YzlSXMN3/7yeX3+4h/JaKWYnRF8miUDYlJub4odTE8n8+VxunJ7EP7dkk7Z0Nf/ckk1zi3QXCdEXSSIQdhHs58lvFo3h47tmMzQykEc+2MOiF9eTlV3m6NCEEB1IIhB2NSo2iHf+azrP/TCF4qp6rnppIz//106KqqRGjRB9hSQCYXdKKRalxJHx87ncMWcwK3bmM2/pGl5dd5TG5pazv4EQwq4kEYhe4+/twcOXjWDlvalMTArlt5/s5/Ln1rHh8ClHhyaES5NEIHrdoIgAXr95Cn/98WTqmpq54dUt/OyfWeSXn3Z0aEK4JEkEwiGUUlwyKoov75vD/ZcMI+NAEfOeXs2fv/pWitkJ0cskEQiH8vF05+55Q1l1/xzShkfy9JeH+M4za/ly30lZnSxEL7FbIlBKLVNKFSml9nTx/A1KqV3mbaNSary9YhF9X3yoHy/9aBL/uHUaXh5u/GT5Nm5+fStHpZidEHZnzxbB68D8bp4/BszRWo8DlgCv2DEW4SRmDQ3ns3tm86sFI9l2vIxLn13LHz4/QI0UsxPCbuyWCLTWa4HSbp7fqLVuXV20GYi3VyzCuXi6u3Hb7EFkPDCH746P5aXVR5j39BpW7CyQ7iIh7KCvjBHcCnzW1ZNKqduVUtuUUtuKi4t7MSzhSJGBPvzpmhTe++kMwgO9uPutb/jhK5vZX1jp6NCE6Ffsunm9UioZ+FhrPaabc9KAvwCztNYlZ3tP2bzeNTW3aN7emsNTKw9SebqRH89I5r6LhxHs5+no0IRwCn1283ql1DjgVWBRT5KAcF3uboobpiWx+oG53DAtieWbjpP29Gre/jqHFilmJ8QFcVgiUEolAu8DN2qtDzkqDuFcQvy8WHLFGD66axaDwv15+P3dXPmXDezILXd0aEI4Lbt1DSml3gLmAuHASeAxwBNAa/2yUupV4Cog23xJU1fNFmvSNSRaaa35cEc+v/v0AMVV9VwzOZ5fzB9BeIC3o0MTos/prmvIrmME9iCJQHRUVdfInzMOs2z9MXy93Ln/kmHcOD0JD/e+MhdCCMfrs2MEQthCoI8n/3P5SD6/N5WUhBB+89E+Fjy/nk1HZNhJiJ6QRCD6jSGRASy/ZSov/2gS1fVNXPfXzdz55nYKK6SYnRDdkUQg+hWlFPPHRPPVz+dwz7yhfLnvJOlL1/Bi5mHqm6SYnRCdkUQg+iUfT3fuu2QYq+6fQ+qwcJ5aeZBLn1lLxoGTjg5NiD5HEoHo1xLC/Pi/Gyez/JapuLkpbnl9G7e+vpXjp2ocHZoQfYYkAuESUodF8Pk9qfzP5SPYfLSE7zyzlqdWHqC2QYrZCSGJQLgMLw83bk8dTMYDc1kwLoYXM41idh/vkmJ2wrVJIhAuJyrIh2euTeHfd8wgxM+LO9/8huv/uoVDJ6scHZoQDiGJQLisKclhfHzXLJYsGs2+wkoue24dT3y0j8q6RkeHJkSvkkQgXJq7m+LGGclkPjCXa6ck8NrGY6QvXc2/tuVKMTvhMiQRCAGE+XvxuyvHsuK/Z5EY5scv3t3F91/ayK48KWYn+j9JBEJYGRsfzLt3zOTpH4wnr+w0i17cwMPv7aKkut7RoQlhN5IIhOjAzU1x1aR4Mh6Yw60XDeTdrDzSlq7mjY3HaWpucXR4QticJAIhuhDk48mvFo7is3tmMzY+mMdW7GXhn9ez5agUsxP9iyQCIc5iaFQg/7h1Gn+5YSKVpxu59pXN3PP2N5yoqHN0aELYhCQCIXpAKcXlY2P46udzuTt9CJ/tOUH606t5ec0RGpqku0g4N0kEQpwDXy937v/OcFbdN4eZg8N58rMDzH92LasPFjk6NCHOmyQCIc5D4gA/Xr1pMq/dPAUNLH5tK7e9sY3PdhdSVCVdRsK5eDg6ACGcWdrwSGYOHsCy9cd5IeNbVu03ylwnhPkyOSmMiUmhTE4KZVhUIO5uysHRCtE52bNYCBupb2pmb0ElWcfLyMouY1t2GafM9QeB3h6kJIYwKSmUSUmhTEgMJcBbfg8TvUc2rxfCAbTW5JaeJiunlG1mcjh4sgqtwU3B8OggJpuJYVJSKPGhviglrQZhHxecCJRS9wCvAVXAq8AE4GGt9Re2DLQnJBEIZ1ZZ18iOnHK2ZZexPbuMb3LKqGkwttCMDPRmcnIoExNDmZwcxqiYILw8ZBhP2EZ3iaCnbdNbtNbPKaUuBSKAmzESQ68nAiGcWZCPJ6nDIkgdFgFAc4vmwIlKtptdSVnZZXy6+wQA3h5ujE8wu5MSjVZDqL+XI8MX/VRPE0Fre/Vy4DWt9U51ljasUmoZsBAo0lqP6eR5BTxnvmctsFhrvb3HkQvRD7i7KUbHBjM6NpgbZyQDcLKyzhhjOF5GVk4Zf117lJfMSqiDIvytupPCGBzhL91J4oL1tGvoNSAOGAiMB9yB1VrrSd28JhWoBpZ3kQguB+7CSATTgOe01tPOFot0DQlXc7qhmV155WTllBkD0TlllNcaeyaE+HkyMbFtnGF8fAi+Xu4Ojlj0RbboGroVSAGOaq1rlVJhGN1DXdJar1VKJXdzyiKMJKGBzUqpEKVUjNa6sIcxCeESfL3cmTZoANMGDQCMQegjxTVmd1IpWdllZBwwFrR5uClGxwYxKSmMSUmhTE4OJSrIx5HhCyfQ00QwA9ihta5RSv0ImIjRrXMh4oBcq8d55rEzEoFS6nbgdoDExMQL/FghnJtSiiGRAQyJDOCaKQkAlNU0sD2nbdrqP7dks2zDMQDiQnyZnNzWahgeFYiHuwxCizY9TQQvAeOVUuOBXwB/A5YDcy7gszvr2Oy0n0pr/QrwChhdQxfwmUL0S6H+XswbGcW8kVEANDS1sK+wkqzsMrKyS9l0pIT/7CgAwN/L3VzTEGauaQghyMfTkeELB+tpImjSWmul1CKMvvy/KaVuusDPzgMSrB7HAwUX+J5CCMDLw42UhBBSEkK4ddZAtNbkl582E4MxEP1Cxre0aFAKhkcFWloMk5JCSQzzk0FoF9LTRFCllPolcCMwWynlDlzorxArgDuVUm9jDBZXyPiAEPahlCI+1I/4UD8WpcQBUF3fxI6ccrM7qZQVOwr455YcAMIDvJmUFGIpkzEmLghvDxmE7q96mgiuBa7HWE9wQimVCDzV3QuUUm8Bc4FwpVQe8Bhm8tBavwx8ijFj6DDG9NFuB5+FELYV4O3BrKHhzBoaDhhrGg6drCLLXOy2LbuMlXuN2kleHm6MiwtmUnLbmoYBAd6ODF/YUI9LTCilooAp5sOvtdYOqbsr00eF6D1FVXVsz24bhN6TX0Fjs/F/xsBwf3MVtJEYhkQE4CaF9fosW5SYuAajBbAaY5B3NvCg1vpdG8bZI5IIhHCcusZmdudXWMYasrLLKK1pACDIx8NSbXViUigpCSH4eUlhvb7CFusIHgGmtLYClFIRwCqg1xOBEMJxfDzdmZIcxpTkMMBY03C8pJZtx0vZnmMMQq8+WAwYq6ZHxQS1G4SODfF1ZPiiCz1NBG4duoJKkE1thHB5SikGhvszMNyfH0w2JgFW1DayPbfMUo77na25vL7xOACxwT6WVsOkpDBGxsiahr6gp4ngc6XUSuAt8/G1GIO9QgjRTrCfJ2nDI0kbHglAY3MLBwqrLKugs7LL+HiXMUHQ19OdlIQQo+pqklF5NdhX1jT0tnMZLL4KuAhjjGCt1voDewbWFRkjEML5FZSftpTizsouY19hJc1mYb1hUQGWonqTkkJJHiBrGmxBNqYRQvRpNfVN7MwrtxTV255dRmVdEwAD/L2supNCGRMXjI+nrGk4V+c9WKyUqqLzsg8K0FrrIBvEJ4Rwcf7eHswcHM7MwcaahpYWzeHiasvOblnZpXy5z1zT4O7GmLigdq2GiEBZ03AhpEUghHAKp6rrLV1JWdll7MqvoKGpBYCkAX7GQjdzTcOwyEBZ09CBdA0JIfqd+qZm9uRXkmU1CH2q2ljTEOjjwYREYxX05ORQxieEEODt2msabLGOQAgh+hRvD3fL+gQw1jTklNZaVkFvzy7j2a8OoTW4KRjZYU1DXIivDEKbpEUghOi3Kk43siO3nKzjpWTllPFNTjm1Dc0ARAf5tEsMo2KD8OzHaxqkRSCEcEnBvp7MGRbBnGERADQ1t3DgRJVlFXRWdhmf7DbWNPh4ujE+PsSys9vExFBC/LwcGX6vkRaBEMKlnaios6qdVMregkqazDUNQyID2g1CDwr3d9ruJBksBg6WHuQvO/5CemI6c+LnEOITYofohBDO7nRDs7GmwWqGUsXpRgBC/TyZZBbVm5wUxrh451nTIF1DwMnak+wp2UNGbgZuyo2JkRNJT0wnLSGN+MB4R4cnhOgjfL3cmT5oANMHDQCMNQ1HT1VbdnbLyilj1X6j9Jqnu2J0bLDRnWSONUQG+Tgy/PPiMi0CMGYV7CvZR0ZuBhk5GRwuPwzAsNBhpCemk56QzoiwEU7b9BNC9I7SmgZjTUOOUVxvZ1459eaahoQwX8vmPZOSwhgeHYh7H1jTIF1DXcitzLUkhR3FO2jRLcT4x5CWkEZ6YjoToybi6SYFsIQQ3WtoamFvgbFPQ+tAdFFVPWDsBDchMcSyiU9KQgiBPr3//4okgh4orStlTe4aMnIz2FSwifrmegK9ApkTP4f0xHQuir0IP08/m3+uEKL/0VqTV3baMsawLbuMgycqaTHXNAyPDmJSkjlDKSmM+FD7r2mQRHCOahtr2VSwiYzcDNbkraGivgIvNy+mx04nPSGdOQlzCPcNt2sMQoj+parOXNNgJodvcsqprjcK60UEelvGGCYlhTI6NhgvD9uuaZBEcAGaWpr4pugbMnIyyMzNJL86H4VifMR4y2BzcnByr8UjhOgfmls0h05WWVZBb8suJbf0NADeHsaaBuutP8P8L2xNgyQCG9Fac6jsEBm5GWTmZLK/dD8Ag4IHWZLCmPAxuKn+uzpRCGE/RZV17bqT9hZU0Nhs/B89KNyfW2YN5EfTk87rvSUR2ElBdQGZuZlk5mSy7eQ2mnUzEb4RpCWkkZaYxtToqXi5u8bKRCGE7dU1NrM7v8KyCvo7o6K4ZkrCeb2XJIJeUFFfwdq8tWTmZrI+fz2nm07j7+nP7LjZpCemMytuFoFegY4OUwjhohyWCJRS84HnAHfgVa31kx2eTwTeAELMcx7WWne7F3JfTQTW6pvr2VK4xTKuUFpXioebB1OjpxqthYQ0ovyjHB2mEMKFOCQRKKXcgUPAJUAesBW4Tmu9z+qcV4BvtNYvKaVGAZ9qrZO7e19nSATWmlua2X1qNxk5GWTkZpBdmQ3AmAFjLOMKg0MGyyI2IYRdOarExFTgsNb6qBnE28AiYJ/VORpo3e4yGIuZMzYAABpRSURBVCiwYzwO4e7mTkpkCimRKdw36T6OVRyzLGJ7/pvnef6b50kMTLQkhfER43F3c47aJUKI/sGeLYKrgfla69vMxzcC07TWd1qdEwN8AYQC/sDFWuusTt7rduB2gMTExEnZ2dl2ibm3FdUWsTp3NRk5GWw5sYWmlibCfMKYmzCXtIQ0psdMx8fD+eqWCCH6Hkd1Df0AuLRDIpiqtb7L6pz7zRieVkrNAP4GjNFat3T1vs7WNdRTVQ1VbMjfQEZOBuvy11HdWI2vhy8XxV5EWmIaqXGpUjFVCHHeHNU1lAdYz3OK58yun1uB+QBa601KKR8gHCiyY1x9UqBXIPMHzmf+wPk0Njey9cRWy3qFVTmrcFfuTIqaZJmaGhcQ5+iQhRD9hD1bBB4Yg8XzgHyMweLrtdZ7rc75DHhHa/26Umok8BUQp7sJqr+2CLrSoluMiqnmDKTWiqkjwkZYiuMNDx0ug81CiG45cvro5cCzGFNDl2mt/1cp9QSwTWu9wpwp9FcgAGPg+Bda6y+6e09XSwQdZVdmk5mTSWZuJt8UfYNGE+sfS1piGukJRsVUDzeX2WZCCNFDsqCsnyo5XcKavDVk5mSysWAjDS0NBHsHkxqXSnpiOjNjZ0rFVCEEIInAJdQ21rKxYCOZuZmszl1NZUMl3u7eTI+Zbtmec4DvAEeHKYRwEEkELqappYntJ7eTmZtJRk4GBTUFKBQpkSmkJ6STnphOYlCio8MUQvQiSQQABTsg83cwZB4MuRjCBoELDLBqrTlYdtAy2Hyg9AAAg4MHG9tzJqYzasAoqZgqRD8niQDg2y/hs19A6VHjcWiykRAGz4OBs8HbNQrC5VfnWwabs05m0aybifSNtAw2T4megqe7bM8pRH8jicBa6VE4/JVxO7YWGmvAzRMSp7e1FqLGuERrobyunLX5a8nMyWRDwQZON50mwDOgXcXUAK8AR4cphLABSQRdaaqHnM1wxEwMJ/cYxwOijaQwON24+YXZ5vP6sLqmOjYXbrYMNrdWTJ0WPY30xHTmJswl0i/S0WEKIc6TJIKeqixsSwpHMqCuHFAQN8loKQyZZ9zv50Xhmlua2Vm8k8zcTL7K+YrcqlwAxoaPNcYVEtIZGDxQFrEJ4UQkEZyPlmbI3w6HVxnJIW8boMEnBAantY0vBMXYPxYH0lpzpPyIZQbSnhKj1ZQUlGSZgTQ2fKxUTBWij5NEYAu1pXA0Ew5nGMmh+oRxPHJ029hC4nTw8O792HrRyZqTRsXU3Ay+LvyaJm1UTG0tdzEtZhre7v37GgjhjCQR2JrWcHJvW2shexO0NIKnHwxMNVsL6TBgsGPjtLOqhirW56+3VEytaazB18OXWXGzSEtIIzU+lWDvYEeHKYRAEoH91VfD8XVGYji8CsqOG8dDB5pjCxdD8izw7r8zcBqaG4yKqeZ6heLTxbgrdyZHTbZMTY0J6N/daEL0ZZIIelvJEXPAuXWKaq0xRTVpRltiiBzVb6eotugW9p7aa9mJ7WiFsXZjZNhIS1IYFjpMBpuF6EWSCBypqR5yNpmthQwoMqtwB8YYg81D5sGguf16iurxiuOWweadxTvRaOIC4izjChMiJ0jFVCHsTBJBX1KRb0xNPbzKGHyuqwDlZjVF9WKIndBvp6ieOn2KNblryMjNYHPBZkvF1DnxcywVU309fB0dphD9jiSCvqq5CQrMKaqHv4L8LECDbygMSmtbuxAY7ehI7aK2sZYNBcb2nGvy1lDVUIW3uzczYmeQnpDOnIQ5hPn035aSEL1JEoGzqC01Wwvm+EL1SeN41Ji2KaoJ08HDy7Fx2kFjSyPbT24nIyeDjNwMTtScwE25kRKRYlnElhCUcPY3EkJ0ShKBM9LaKHnR2lrI2WxOUfU3p6i2VlEd6OhIbU5rzYHSA5Y9mw+WHQRgSMgQS1IYNWCUDDYLcQ4kEfQH9VVwzGqKanm2cTxscFtSSJ4FXv6OjdMO8qryyMxtq5jaoluI8osiLSGNtMQ0pkRJxVQhzkYSQX+jtVlF1UwKx9ZB02lw94KkmeZspIshcmS/m6JaVlfG2ry1ZOZmsiF/A3XNdQR6BjI7fjZpiWnMipWKqUJ0RhJBf9dYZzVF9Sso3m8cD4yFIelGUhg01xiE7kdON51mc8FmMnIzWJO7hrL6MjzdPJkWM81oLSSkEeEX4egwhegTJBG4mop8s4rqKjiyGurNKarxU9paC7Ep/WqKanNLMzuKdxiDzTkZ5FXnATAuYpxlvcKg4EEOjlIIx5FE4Mqam4xpqa3dSAXfYExRDTPqIQ2ZZySHwChHR2ozWmsOlx+2lLvYW2Is4ksOSrasbB4XMU625xQuxWGJQCk1H3gOcAde1Vo/2ck51wCPAxrYqbW+vrv3lERwgWpKzCqqZjdSTZFxPHpsW2nthGn9aorqiZoTxmBzTiZbT2ylSTcxwGcAcxPmSsVU4TIckgiUUu7AIeASIA/YClyntd5ndc5Q4F9Auta6TCkVqbUu6u59JRHYUEtL+ymquZuhpQm8AmDgnLbxhdBkR0dqM5UNlazLW0dmbibr8tZR21SLn4cfF8VdRHpiOrPjZkvFVNEvOSoRzAAe11pfaj7+JYDW+vdW5/wROKS1frWn7yuJwI7qKttXUS3PMY4PGNLWWkieBV5+jo3TRhqaG9hSuMUyNfXU6VN4KA8mR0+2jCtE+/fPVd3C9TgqEVwNzNda32Y+vhGYprW+0+qcDzFaDRdhdB89rrX+vLv3lUTQS7SGksNGS+HwKji+3pyi6m1MUW0tfxExol9MUW3RLew+tZvMnEwycjM4VnEMgFEDRlmSwtCQobKITTgtRyWCHwCXdkgEU7XWd1md8zHQCFwDxAPrgDFa6/IO73U7cDtAYmLipOzsbLvELLrRWAc5G9sSQ/EB43hQXNuA86C54BviyCht5ljFMUvF1F3Fu9Bo4gPiLYPNEyInyPacwqn05a6hl4HNWuvXzcdfAQ9rrbd29b7SIugjKvLaksLR1VBfCcrdmKLa2lqISQE355+Zc+r0KWN7zpwMNhduprGlkVDvUFLjU0lPTGdG7AypmCr6PEclAg+Mbp95QD7GYPH1Wuu9VufMxxhAvkkpFQ58A6RorUu6el9JBH1QcyPkbWtbu1DwjXHcb4A5RdXcujMg0rFx2kBNYw0b8jeQkZvB2ty1VDVW4ePuw4zYGaQlpDE0dCgx/jGE+YRJN5LoUxw5ffRy4FmM/v9lWuv/VUo9AWzTWq9Qxr+Up4H5QDPwv1rrt7t7T0kETqDmFBzJbNvTuabYOB49rq21kDANnLw+UGNLI1knsyyL2E7WnrQ85+PuQ0xADLH+se3+jAuII8Y/hgjfCOlaEr1KFpQJx2lpgRO7zNbCV5C7xZyiGgiD5rSNL4QmOTrSC6K15kj5EXKqciisKaSguoDCmkLyq/MprC6krL6s3fkeyoMo/yhiA2KJ8W9LELEBscT6xxLtHy2F9IRNSSIQfUddhbGP82EzMVS0TlEd2rZDW/JF4Nm/+txrG2s5UXOCgpqCM5JEQU0BxbXFaNr+LSoUEb4RRmvCTA6tSaP1Tz/P/jGNV/QOSQSib9IaTn3bNrZwfD001RlTVJMvaksM4cP6xRTV7jQ2N3Ki9gSF1WaCsGpVFFQXcKL2BE0tTe1eE+od2m33U5BXkIxTCAtJBMI5NJ6G7A1w2NzT+ZSxIQ1B8W17LgyaAz6ut/K3uaWZ4tPFZyQISwujupC65rp2r/H39G/XgogNiG3XuhjgM0AShQuRRCCcU3lO27adR9e0TVFNmNqWGKLH94spqhdKa01ZfZmlq6mz7qeqhqp2r/Fy8yImIOaMMYrWPyP9IvFw83DQTyRsTRKBcH7NjZC3ta0uUuEO47hfeIcpqrL/QFeqG6opqCnosvuppK79rG135U6UX1SX3U/R/tFSrM+JSCIQ/U91MRwxu5COZEDtKeN4zPi2sYX4KU4/RbU31TXVUVhTeEarovXPk7UnadEt7V4T7hve5RhFbEAs/p79b+tUZyWJQPRvLS1wYqfZWsgwpqjqZvAOgoGpbWsXQhIdHalTa2xppKi26IwE0dr9VFhTSGNLY7vXBHkFdTpFtjVxhHiHyDhFL5FEIFxLXYUxptDajVRp7FZG+LC2pJDU/6aoOlqLbqHkdIml+6m1VWGdOGqbatu9xtfD94wWRetgdmxALOG+4bKBkI1IIhCuS2s4daittPbxDdBcDx4+RjKwTFEd2u+nqDqa1prKhkpLcuis+6m8vl29STzdPIn2j+6y+ynKPwpPN+n+6wlJBEK0aqiF7I1tiaHkW+N4cGLbRjwD54BPkGPjdFG1jbWWJNHaqrBuXRSfLm53vptyI8I3on33k1XCiPGPkYKAJkkEQnSlLLut/MXRNdBQBW4eRi2k1tlI0eNkimof0dDcYFmh3Vn308makzTp9gvvwnzC2pXvsO5+igkwFt65AkkEQvREcyPkft3WWjixyzjuH9F+iqp/uGPjFF1qXXhXUF3Q6RTZwppC6pvr270m0DOw2xXa/aWSrCQCIc5HdVGHKaolgILYFKNQnmWKqiy6chZaa0rrSrvtfqpurG73Gh93H2OcopPup9iAWKepJCuJQIgL1dJiLGJr3Ywnb6s5RTXYqopqOgQnyKCzk6tsqDQSQ4dk0dqiKK0rbXd+x0qyHbugov2j8XL3ctBP00YSgRC2drocjllPUc03jnv4QGAMBMUat3b3YyEoBgKipRXhxE43nT5j4Z1191NRbVHXlWS76H7qjUqykgiEsCetofigUV67PBuqCqGy0EgOVYXQ3NDhBQoCooykEGgmCcv9GGMf6MAY8A5wyI8jLkxjSyMna0522f1UWFN4RiXZEO+QdrWeOnY/2aKSrCQCIRxFa6gthaoCqDRvVWaSqCw07xdAXfmZr/UOat+qOKOlEWdsByozmpxKi26huLa4XQXZwupC8mvaVmifbjrd7jV+Hn7EBsTyg2E/4PqR15/X53aXCKR9KoQ9KQX+A4xb9Niuz2uobUsKlQVm4ihsSyBHDkL1CehQ6wc3TzMpxLTvfrK+HxgDHlIcrq9wU25E+UcR5R9FSmTKGc9rrSmvL29rRVgtvrPXmghJBEL0BV5+MGCwcetKcxPUFFklCKvup8oCKNwFh1ZCY+2Zr/UL79AVFWuVQMyuKJ9gGejuA5RShPqEEuoTyugBo3vlMyURCOEs3D3a/hNnUufnaG3UWjqj+ym/LYHkZ7VVa7Xm6ddJV1Rc+wQSEAlOMFVSnBtJBEL0J0qBb4hxixzZ9XlN9WcOalt3RWVvMo51qCaKcm8b6G7XFRXXPoF4yX7KzkQSgRCuyMMbQpONW1daWoyWg2WAu6D9/eJDbTvHdeQT0sUAt9VUWr8w6YrqIyQRCCE65+ZmdAUFRAJnDmpa1Fe1H7ewzJAy75/cC9UngQ4zFN2920+b7awrKjBaNhfqBXZNBEqp+cBzgDvwqtb6yS7Ouxr4NzBFay1zQ4VwJt6BEBEIEcO6Pqe50UgGHQe4W//M327cb6rr8EJl1Hpq1/3USVeUVIu9IHZLBEopd+BF4BIgD9iqlFqhtd7X4bxA4G5gi71iEUI4mLsnBMcbN6Z0fo7WcLqsk64os3VRlg05m4xzOvIK7KRV0aEryj9C1lx0wZ4tgqnAYa31UQCl1NvAImBfh/OWAH8EHrBjLEKIvk4pY9zALwyix3R9XuNpq0RReOZivWNroOqEUQvKmpuHkRxaWxVnTKU1//T0se/P2QfZMxHEAblWj/OAadYnKKUmAAla64+VUl0mAqXU7cDtAImJsu+sEC7N0xfCBhm3rrQ0Q03xmQPcrS2Mk/vg21XQWHPma33Duq8VFRgDvqH9aqDbnomgs6tkGS1SSrkBzwCLz/ZGWutXgFfAKDFho/iEEP2Vm7sx0BwYDXETOz9Ha2PGU2cD3K1jGQXfGAmlIw/fs9eKCohymuKC9owyD0iwehwPFFg9DgTGAKvNYkrRwAql1PdkwFgIYXdKGaupfYIhckTX5zU1GC2KjgPcrfdzt3ReXFC5Gcmgu1pRQTHg5W/fn7MH7JkItgJDlVIDgXzgh4ClWpLWugKwbPWklFoNPCBJQAjRp3h4QWiSceuK1sbGRR0HuFvvlxyB4+uMVd8deQefpVZUrN2LC9otEWitm5RSdwIrMaaPLtNa71VKPQFs01qvsNdnCyFEr1LK2MLUPxxixnV9XkNN5wPcrfeL9hvTbDsWF3T3Mrq5pv4XzLzT5uHbtQNLa/0p8GmHY492ce5ce8YihBAO5+UP4UOMW1eam4xk0K4ryqwVFRBll7CcYyRDCCFchbsHBMcZt14iqyuEEMLFSSIQQggXJ4lACCFcnCQCIYRwcZIIhBDCxUkiEEIIFyeJQAghXJwkAiGEcHFKa+cq5qmUKgayz/Pl4cApG4ZjK301Lui7sUlc50biOjf9Ma4krXVEZ084XSK4EEqpbVrryY6Oo6O+Ghf03dgkrnMjcZ0bV4tLuoaEEMLFSSIQQggX52qJ4BVHB9CFvhoX9N3YJK5zI3GdG5eKy6XGCIQQQpzJ1VoEQgghOpBEIIQQLq7fJAKl1Hyl1EGl1GGl1MOdPO+tlHrHfH6LUirZ6rlfmscPKqUu7eW47ldK7VNK7VJKfaWUSrJ6rlkptcO82XRrzx7EtVgpVWz1+bdZPXeTUupb83ZTL8f1jFVMh5RS5VbP2fN6LVNKFSml9nTxvFJKPW/GvUspNdHqOXter7PFdYMZzy6l1Eal1Hir544rpXab18ume4X3IK65SqkKq7+vR62e6/Y7YOe4HrSKaY/5nQozn7PL9VJKJSilMpVS+5VSe5VS93Ryjn2/X1prp79h7Il8BBgEeAE7gVEdzvkZ8LJ5/4fAO+b9Ueb53sBA833cezGuNMDPvP/T1rjMx9UOvF6LgRc6eW0YcNT8M9S8H9pbcXU4/y6MvbDter3M904FJgJ7unj+cuAzQAHTgS32vl49jGtm6+cBl7XGZT4+DoQ76HrNBT6+0O+ArePqcO53gQx7Xy8gBpho3g8EDnXy79Gu36/+0iKYChzWWh/VWjcAbwOLOpyzCHjDvP8uME8ppczjb2ut67XWx4DD5vv1Slxa60ytda35cDMQb6PPvqC4unEp8KXWulRrXQZ8Ccx3UFzXAW/Z6LO7pbVeC5R2c8oiYLk2bAZClFIx2Pd6nTUurfVG83Oh975fPbleXbmQ76at4+qV75fWulBrvd28XwXsBzruU2nX71d/SQRxQK7V4zzOvJCWc7TWTUAFMKCHr7VnXNZuxcj6rXyUUtuUUpuVUlfYKKZziesqsxn6rlIq4Rxfa8+4MLvQBgIZVoftdb16oqvY7Xm9zlXH75cGvlBKZSmlbndAPDOUUjuVUp8ppUabx/rE9VJK+WH8h/qe1WG7Xy9ldFlPALZ0eMqu36/+snm96uRYx3mxXZ3Tk9eerx6/t1LqR8BkYI7V4UStdYFSahCQoZTarbU+0ktxfQS8pbWuV0rdgdGaSu/ha+0ZV6sfAu9qrZutjtnrevWEI75fPaaUSsNIBLOsDl9kXq9I4Eul1AHzN+besB2j9k21Uupy4ENgKH3kemF0C23QWlu3Hux6vZRSARiJ516tdWXHpzt5ic2+X/2lRZAHJFg9jgcKujpHKeUBBGM0EXvyWnvGhVLqYuAR4Hta6/rW41rrAvPPo8BqjN8UeiUurXWJVSx/BSb19LX2jMvKD+nQbLfj9eqJrmK35/XqEaXUOOBVYJHWuqT1uNX1KgI+wHZdomelta7UWleb9z8FPJVS4fSB62Xq7vtl8+ullPLESAL/1Fq/38kp9v1+2XrgwxE3jJbNUYyugtYBptEdzvlv2g8W/8u8P5r2g8VHsd1gcU/imoAxODa0w/FQwNu8Hw58i40GzXoYV4zV/SuBzbptcOqYGV+oeT+st+IyzxuOMXCneuN6WX1GMl0Pfi6g/WDe1/a+Xj2MKxFj3Gtmh+P+QKDV/Y3A/F6MK7r17w/jP9Qc89r16Dtgr7jM51t/SfTvjetl/tzLgWe7Oceu3y+bXVxH3zBG1Q9h/Kf6iHnsCYzfsgF8gH+b/yi+BgZZvfYR83UHgct6Oa5VwElgh3lbYR6fCew2/yHsBm7t5bh+D+w1Pz8TGGH12lvM63gYuLk34zIfPw482eF19r5ebwGFQCPGb2G3AncAd5jPK+BFM+7dwOReul5ni+tVoMzq+7XNPD7IvFY7zb/nR3o5rjutvl+bsUpUnX0Heisu85zFGBNIrF9nt+uF0V2ngV1Wf0+X9+b3S0pMCCGEi+svYwRCCCHOkyQCIYRwcZIIhBDCxUkiEEIIFyeJQAghXJwkAiFMHaqX7rBl5UulVHJXFS+FcLT+UmJCCFs4rbVOcXQQQvQ2aREIcRZmHfo/KKW+Nm9DzONJythDonUviUTzeJRS6gOzoNpOpdRM863clVJ/NWvOf6GU8jXPv1u17UnxtoN+TOHCJBEI0ca3Q9fQtVbPVWqtpwIvAM+ax17AKA08Dvgn8Lx5/HlgjdZ6PEbt+73m8aHAi1rr0UA5cJV5/GFggvk+d9jrhxOiK7KyWAiTUqpaax3QyfHjQLrW+qhZHOyE1nqAUuoURk2mRvN4odY6XClVDMRrqwKCZnnhL7XWQ83HDwGeWuvfKqU+B6oxKnB+qM1ibEL0FmkRCNEzuov7XZ3TmXqr+820jdEtwKgjMwnIMqvjCtFrJBEI0TPXWv25yby/EaOSLcANwHrz/lcY246ilHJXSgV19aZKKTcgQWudCfwCCAHOaJUIYU/ym4cQbXyVUjusHn+utW6dQuqtlNqC8cvTdeaxu4FlSqkHgWLgZvP4PcArSqlbMX7z/ylGxcvOuAP/UEoFY1SYfEZrXW6zn0iIHpAxAiHOwhwjmKy1PuXoWISwB+kaEkIIFyctAiGEcHHSIhBCCBcniUAIIVycJAIhhHBxkgiEEMLFSSIQQggX9//BATFA+8PcgQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_graphs(history, 'loss', 'output_1_loss', 'output_2_loss')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyMn6I90a+EqoM9Ks6eBcRWt",
   "collapsed_sections": [],
   "name": "KorQuad_class.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "1fde406732df4b5b90b7701dc7e4981e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2c0ecef646d44a0580cacefa5c3fd9f2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "3be3c9704e934fb5a3d5847749d398ce": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a7d4d0c48cda4abdb106a6bcfb24359e",
      "placeholder": "​",
      "style": "IPY_MODEL_f58154a65f974e04bcf8af24b2884fdd",
      "value": " 872k/872k [00:00&lt;00:00, 3.17MB/s]"
     }
    },
    "a7d4d0c48cda4abdb106a6bcfb24359e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bc7f3c579a324f77811bdd6ad6dd7dc0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f8f80478dfca4894ac1ff8c2a082f734",
       "IPY_MODEL_3be3c9704e934fb5a3d5847749d398ce"
      ],
      "layout": "IPY_MODEL_e31de13423d743e68d6c451d23c93cdf"
     }
    },
    "e31de13423d743e68d6c451d23c93cdf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f58154a65f974e04bcf8af24b2884fdd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f8f80478dfca4894ac1ff8c2a082f734": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1fde406732df4b5b90b7701dc7e4981e",
      "max": 871891,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2c0ecef646d44a0580cacefa5c3fd9f2",
      "value": 871891
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}