7.PRETRAIN_METHOD/7.4.3.gpt2_finetune_KorNLI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from transformers import TFGPT2Model\n",
    "\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "import gluonnlp as nlp\n",
    "from gluonnlp.data import SentencepieceTokenizer\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "아레 실행 커멘드는 gpt_ckpt 폴더가 있지 않은 경우에만 실행해주세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wget\n",
    "import zipfile\n",
    "\n",
    "wget.download('https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/releases/download/v1.0/gpt_ckpt.zip')\n",
    "\n",
    "with zipfile.ZipFile('gpt_ckpt.zip') as z:\n",
    "    z.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 시각화\n",
    "\n",
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history['val_'+string], '')\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, 'val_'+string])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED_NUM = 1234\n",
    "tf.random.set_seed(SEED_NUM)\n",
    "np.random.seed(SEED_NUM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 데이터 준비하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOKENIZER_PATH = './gpt_ckpt/gpt2_kor_tokenizer.spiece'\n",
    "\n",
    "tokenizer = SentencepieceTokenizer(TOKENIZER_PATH)\n",
    "vocab = nlp.vocab.BERTVocab.from_sentencepiece(TOKENIZER_PATH,\n",
    "                                               mask_token=None,\n",
    "                                               sep_token='<unused0>',\n",
    "                                               cls_token=None,\n",
    "                                               unknown_token='<unk>',\n",
    "                                               padding_token='<pad>',\n",
    "                                               bos_token='<s>',\n",
    "                                               eos_token='</s>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "NUM_EPOCHS = 3\n",
    "SENT_MAX_LEN = 31\n",
    "\n",
    "DATA_IN_PATH = './data_in/KOR'\n",
    "DATA_OUT_PATH = \"./data_out/KOR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Train dataset\n",
    "\n",
    "TRAIN_SNLI_DF = os.path.join(DATA_IN_PATH, 'KorNLI', 'snli_1.0_train.kor.tsv')\n",
    "TRAIN_XNLI_DF = os.path.join(DATA_IN_PATH, 'KorNLI', 'multinli.train.ko.tsv')\n",
    "DEV_XNLI_DF = os.path.join(DATA_IN_PATH, 'KorNLI', 'xnli.dev.ko.tsv')\n",
    "\n",
    "train_data_snli = pd.read_csv(TRAIN_SNLI_DF, header=0, delimiter='\\t', quoting=3)\n",
    "train_data_xnli = pd.read_csv(TRAIN_XNLI_DF, header=0, delimiter='\\t', quoting=3)\n",
    "dev_data_xnli = pd.read_csv(DEV_XNLI_DF, header=0, delimiter='\\t', quoting=3)\n",
    "\n",
    "train_data_snli_xnli = train_data_snli.append(train_data_xnli)\n",
    "train_data_snli_xnli = train_data_snli_xnli.dropna()\n",
    "train_data_snli_xnli = train_data_snli_xnli.reset_index()\n",
    "\n",
    "dev_data_xnli = dev_data_xnli.dropna()\n",
    "\n",
    "print(\"Total # dataset: train - {}, dev - {}\".format(len(train_data_snli_xnli), len(dev_data_xnli)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 토크나이저를 제외하고는 5장에서 처리한 방식과 유사하게 접근\n",
    "def clean_text(sent):\n",
    "    sent_clean = re.sub(\"[^가-힣ㄱ-ㅎㅏ-ㅣ\\\\s]\", \" \", sent)\n",
    "    return sent_clean\n",
    "\n",
    "train_data_sents = []\n",
    "\n",
    "for train_sent_1, train_sent_2 in train_data_snli_xnli[['sentence1', 'sentence2']].values:\n",
    "    train_tokenized_sent_1 = vocab[tokenizer(clean_text(train_sent_1))]\n",
    "    train_tokenized_sent_2 = vocab[tokenizer(clean_text(train_sent_2))]\n",
    "\n",
    "    tokens = [vocab[vocab.bos_token]] \n",
    "    tokens += pad_sequences([train_tokenized_sent_1], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.sep_token]]  \n",
    "    tokens += pad_sequences([train_tokenized_sent_2], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.eos_token]]\n",
    "\n",
    "    train_data_sents.append(tokens)    \n",
    "\n",
    "train_data_sents = np.array(train_data_sents, dtype=np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data_sents = []\n",
    "\n",
    "for dev_sent_1, dev_sent_2 in dev_data_xnli[['sentence1', 'sentence2']].values:\n",
    "    dev_tokenized_sent_1 = vocab[tokenizer(clean_text(dev_sent_1))]\n",
    "    dev_tokenized_sent_2 = vocab[tokenizer(clean_text(dev_sent_2))]\n",
    "\n",
    "    tokens = [vocab[vocab.bos_token]] \n",
    "    tokens += pad_sequences([dev_tokenized_sent_1], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.sep_token]]  \n",
    "    tokens += pad_sequences([dev_tokenized_sent_2], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.eos_token]]\n",
    "\n",
    "    dev_data_sents.append(tokens)    \n",
    "\n",
    "dev_data_sents = np.array(dev_data_sents, dtype=np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Label을 Netural, Contradiction, Entailment 에서 숫자 형으로 변경한다.\n",
    "label_dict = {\"entailment\": 0, \"contradiction\": 1, \"neutral\": 2}\n",
    "\n",
    "def convert_int(label):\n",
    "    num_label = label_dict[label]    \n",
    "    return num_label\n",
    "\n",
    "train_data_snli_xnli[\"gold_label_int\"] = train_data_snli_xnli[\"gold_label\"].apply(convert_int)\n",
    "train_data_labels = np.array(train_data_snli_xnli['gold_label_int'], dtype=int)\n",
    "\n",
    "dev_data_xnli[\"gold_label_int\"] = dev_data_xnli[\"gold_label\"].apply(convert_int)\n",
    "dev_data_labels = np.array(dev_data_xnli['gold_label_int'], dtype=int)\n",
    "\n",
    "print(\"# train labels: {}, #dev labels: {}\".format(len(train_data_labels), len(dev_data_labels)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모델 학습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFGPT2Classifier(tf.keras.Model):\n",
    "    def __init__(self, dir_path, num_class):\n",
    "        super(TFGPT2Classifier, self).__init__()\n",
    "        \n",
    "        self.gpt2 = TFGPT2Model.from_pretrained(dir_path)\n",
    "        self.num_class = num_class\n",
    "        \n",
    "        self.dropout = tf.keras.layers.Dropout(self.gpt2.config.summary_first_dropout)\n",
    "        self.classifier = tf.keras.layers.Dense(self.num_class, \n",
    "                                                kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=self.gpt2.config.initializer_range), \n",
    "                                                name=\"classifier\")\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        outputs = self.gpt2(inputs)\n",
    "        pooled_output = outputs[0][:, -1]\n",
    "\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        logits = self.classifier(pooled_output)\n",
    "\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODEL_PATH = './gpt_ckpt'\n",
    "sim_model = TFGPT2Classifier(dir_path=BASE_MODEL_PATH, num_class=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(6.25e-5)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
    "sim_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"tf2_gpt_kornli\"\n",
    "\n",
    "earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=2)\n",
    "\n",
    "checkpoint_path = os.path.join(DATA_OUT_PATH, model_name, 'weights.h5')\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "if os.path.exists(checkpoint_dir):\n",
    "    print(\"{} -- Folder already exists \\n\".format(checkpoint_dir))\n",
    "else:\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    print(\"{} -- Folder create complete \\n\".format(checkpoint_dir))\n",
    "    \n",
    "cp_callback = ModelCheckpoint(\n",
    "    checkpoint_path, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "history = sim_model.fit(train_data_sents, train_data_labels, \n",
    "                        epochs=NUM_EPOCHS,\n",
    "                        validation_data=(dev_data_sents, dev_data_labels),\n",
    "                        batch_size=BATCH_SIZE, \n",
    "                        callbacks=[earlystop_callback, cp_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plot_graphs(history, 'loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모델 테스트"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Load Test dataset\n",
    "TEST_XNLI_DF = os.path.join(DATA_IN_PATH, 'KorNLI', 'xnli.test.ko.tsv')\n",
    "\n",
    "test_data_xnli = pd.read_csv(TEST_XNLI_DF, header=0, delimiter='\\t', quoting=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_data_xnli = test_data_xnli[:50] # for test\n",
    "\n",
    "test_data_sents = []\n",
    "\n",
    "for test_sent_1, test_sent_2 in test_data_xnli[['sentence1', 'sentence2']].values:\n",
    "    test_tokenized_sent_1 = vocab[tokenizer(clean_text(test_sent_1))]\n",
    "    test_tokenized_sent_2 = vocab[tokenizer(clean_text(test_sent_2))]\n",
    "\n",
    "    tokens = [vocab[vocab.bos_token]] \n",
    "    tokens += pad_sequences([test_tokenized_sent_1], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.sep_token]]  \n",
    "    tokens += pad_sequences([test_tokenized_sent_2], \n",
    "                            SENT_MAX_LEN, \n",
    "                            value=vocab[vocab.padding_token], \n",
    "                            padding='post').tolist()[0] \n",
    "    tokens += [vocab[vocab.eos_token]]\n",
    "\n",
    "    test_data_sents.append(tokens)    \n",
    "\n",
    "test_data_sents = np.array(test_data_sents, dtype=np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_xnli[\"gold_label_int\"] = test_data_xnli[\"gold_label\"].apply(convert_int)\n",
    "test_data_labels = np.array(test_data_xnli['gold_label_int'], dtype=int)\n",
    "\n",
    "print(\"# sents: {}, # labels: {}\".format(len(test_data_sents), len(test_data_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_model.load_weights(checkpoint_path)\n",
    "\n",
    "results = sim_model.evaluate(test_data_sents, test_data_labels, batch_size=1024)\n",
    "print(\"test loss, test acc: \", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}