updated by new feedbacks on 20201218

Author: USER

4.TEXT_CLASSIFICATION/4.1.4 Linear Regression with Word2Vec.ipynb

@@ -38,7 +38,6 @@
    "outputs": [],
    "source": [
     "DATA_IN_PATH = './data_in/'\n",
-    "DATA_OUT_PATH = './data_out/'\n",
     "TRAIN_CLEAN_DATA = 'train_clean.csv'\n",
     "\n",
     "RANDOM_SEED = 42\n",
@@ -192,18 +191,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "predicted = lgs.predict(X_test)\n",
-    "from sklearn import metrics\n",
-    "\n",
-    "fpr, tpr, _ = metrics.roc_curve(y_test, (lgs.predict_proba(X_test)[:, 1]))\n",
-    "auc = metrics.auc(fpr, tpr)\n",
-    "\n",
-    "print(\"------------\")\n",
-    "print(\"Accuracy: %f\" % lgs.score(X_test, y_test))  #checking the accuracy\n",
-    "print(\"Precision: %f\" % metrics.precision_score(y_test, predicted))\n",
-    "print(\"Recall: %f\" % metrics.recall_score(y_test, predicted))\n",
-    "print(\"F1-Score: %f\" % metrics.f1_score(y_test, predicted))\n",
-    "print(\"AUC: %f\" % auc)"
+    "print(\"Accuracy: %f\" % lgs.score(X_test, y_test)) "
    ]
   },
   {
@@ -254,18 +242,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "test_predicted = lgs.predict(test_data_vecs)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ids = list(test_data['id'])\n",
+    "DATA_OUT_PATH = './data_out/'\n",
+    "\n",
+    "test_predicted = lgs.predict(test_data_vecs)\n",
     "\n",
-    "answer_dataset = pd.DataFrame({'id': ids, 'sentiment': test_predicted})"
+    "if not os.path.exists(DATA_OUT_PATH):\n",
+    "    os.makedirs(DATA_OUT_PATH)\n",
+    "    \n",
+    "ids = list(test_data['id'])\n",
+    "answer_dataset = pd.DataFrame({'id': ids, 'sentiment': test_predicted})\n",
+    "answer_dataset.to_csv(DATA_OUT_PATH + 'lgs_w2v_answer.csv', index=False, quoting=3)"
    ]
   },
   {
@@ -274,10 +260,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "if not os.path.exists(DATA_OUT_PATH):\n",
-    "    os.makedirs(DATA_OUT_PATH)\n",
-    "\n",
-    "answer_dataset.to_csv(DATA_OUT_PATH + 'lgs_w2v_answer.csv', index=False, quoting=3)"
+    "model_name = \"300features_40minwords_10context\"\n",
+    "model.save(model_name)"
    ]
   }
  ],
@@ -297,7 +281,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.8"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,

7.PRETRAIN_METHOD/7.4.1.gpt2_finetune_LM.ipynb

@@ -33,8 +33,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!wget https://www.dropbox.com/s/nzfa9xpzm4edp6o/gpt_ckpt.zip -O gpt_ckpt.zip\n",
-    "!unzip -o gpt_ckpt.zip"
+    "import wget\n",
+    "import zipfile\n",
+    "\n",
+    "wget.download('https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/releases/download/v1.0/gpt_ckpt.zip')\n",
+    "\n",
+    "with zipfile.ZipFile('gpt_ckpt.zip') as z:\n",
+    "    z.extractall()"
    ]
   },
   {
@@ -157,7 +162,7 @@
     "DATA_IN_PATH = './data_in/KOR/'\n",
     "TRAIN_DATA_FILE = 'finetune_data.txt'\n",
     "\n",
-    "sents = [s[:-1] for s in open(DATA_IN_PATH + TRAIN_DATA_FILE).readlines()]"
+    "sents = [s[:-1] for s in open(DATA_IN_PATH + TRAIN_DATA_FILE, encoding='utf-8').readlines()]"
    ]
   },
   {
@@ -286,7 +291,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.9"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,

7.PRETRAIN_METHOD/7.4.2.gpt2_finetune_NSMC.ipynb

@@ -38,8 +38,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!wget https://www.dropbox.com/s/nzfa9xpzm4edp6o/gpt_ckpt.zip -O gpt_ckpt.zip\n",
-    "!unzip -o gpt_ckpt.zip"
+    "import wget\n",
+    "import zipfile\n",
+    "\n",
+    "wget.download('https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/releases/download/v1.0/gpt_ckpt.zip')\n",
+    "\n",
+    "with zipfile.ZipFile('gpt_ckpt.zip') as z:\n",
+    "    z.extractall()"
    ]
   },
   {
@@ -365,7 +370,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.1"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,

7.PRETRAIN_METHOD/7.4.3.gpt2_finetune_KorNLI.ipynb

@@ -36,8 +36,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!wget https://www.dropbox.com/s/nzfa9xpzm4edp6o/gpt_ckpt.zip -O gpt_ckpt.zip\n",
-    "!unzip -o gpt_ckpt.zip"
+    "import wget\n",
+    "import zipfile\n",
+    "\n",
+    "wget.download('https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/releases/download/v1.0/gpt_ckpt.zip')\n",
+    "\n",
+    "with zipfile.ZipFile('gpt_ckpt.zip') as z:\n",
+    "    z.extractall()"
    ]
   },
   {
@@ -424,7 +429,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.6"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,

7.PRETRAIN_METHOD/7.4.4.gpt2_finetune_KorSTS.ipynb

@@ -37,8 +37,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!wget https://www.dropbox.com/s/nzfa9xpzm4edp6o/gpt_ckpt.zip -O gpt_ckpt.zip\n",
-    "!unzip -o gpt_ckpt.zip"
+    "import wget\n",
+    "import zipfile\n",
+    "\n",
+    "wget.download('https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/releases/download/v1.0/gpt_ckpt.zip')\n",
+    "\n",
+    "with zipfile.ZipFile('gpt_ckpt.zip') as z:\n",
+    "    z.extractall()"
    ]
   },
   {
@@ -494,7 +499,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.1"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,