Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Importing Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T15:24:53.492926Z",
     "iopub.status.busy": "2021-12-13T15:24:53.492433Z",
     "iopub.status.idle": "2021-12-13T15:24:55.444548Z",
     "shell.execute_reply": "2021-12-13T15:24:55.443802Z",
     "shell.execute_reply.started": "2021-12-13T15:24:53.492877Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/ucloud/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/ucloud/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/ucloud/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/ucloud/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "# Load packages for data wrangling:\n",
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Load packages for fine-tuning BERT model:\n",
    "from simpletransformers.classification import ClassificationModel\n",
    "\n",
    "# Load scikit-learn train_test_split:\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Load classification metrics:\n",
    "from sklearn.metrics import (accuracy_score, recall_score, precision_score, f1_score,\n",
    "                            classification_report,confusion_matrix)\n",
    "\n",
    "# Load softmax for converting raw model outpus to probabilities:\n",
    "from scipy.special import softmax\n",
    "\n",
    "# Load packages for data cleaning:\n",
    "import string\n",
    "import re\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import pos_tag\n",
    "from nltk import sent_tokenize, word_tokenize\n",
    "\n",
    "# Set stopword corpus\n",
    "stopword = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "# Set NLTK lemmatizer\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Preprocessing of dataset 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Data loading and wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T15:24:56.333789Z",
     "iopub.status.busy": "2021-12-13T15:24:56.333301Z",
     "iopub.status.idle": "2021-12-13T15:24:57.369683Z",
     "shell.execute_reply": "2021-12-13T15:24:57.368622Z",
     "shell.execute_reply.started": "2021-12-13T15:24:56.333739Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Loading data with fake news as pandas dataframe:\n",
    "fake_df = pd.read_csv(os.path.join(\"data\", \"dataset_1\", \"Fake.csv\"))\n",
    "\n",
    "# Loading data with true news as pandas dataframe:\n",
    "true_df = pd.read_csv(os.path.join(\"data\", \"dataset_1\", \"True.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T15:24:57.382478Z",
     "iopub.status.busy": "2021-12-13T15:24:57.382353Z",
     "iopub.status.idle": "2021-12-13T15:24:57.396390Z",
     "shell.execute_reply": "2021-12-13T15:24:57.395885Z",
     "shell.execute_reply.started": "2021-12-13T15:24:57.382463Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>subject</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Donald Trump Sends Out Embarrassing New Year’...</td>\n",
       "      <td>Donald Trump just couldn t wish all Americans ...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 31, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Drunk Bragging Trump Staffer Started Russian ...</td>\n",
       "      <td>House Intelligence Committee Chairman Devin Nu...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 31, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sheriff David Clarke Becomes An Internet Joke...</td>\n",
       "      <td>On Friday, it was revealed that former Milwauk...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 30, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Trump Is So Obsessed He Even Has Obama’s Name...</td>\n",
       "      <td>On Christmas day, Donald Trump announced that ...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 29, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pope Francis Just Called Out Donald Trump Dur...</td>\n",
       "      <td>Pope Francis used his annual Christmas Day mes...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 25, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Racist Alabama Cops Brutalize Black Boy While...</td>\n",
       "      <td>The number of cases of cops brutalizing and ki...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 25, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Fresh Off The Golf Course, Trump Lashes Out A...</td>\n",
       "      <td>Donald Trump spent a good portion of his day a...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 23, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Trump Said Some INSANELY Racist Stuff Inside ...</td>\n",
       "      <td>In the wake of yet another court decision that...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 23, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Former CIA Director Slams Trump Over UN Bully...</td>\n",
       "      <td>Many people have raised the alarm regarding th...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 22, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>WATCH: Brand-New Pro-Trump Ad Features So Muc...</td>\n",
       "      <td>Just when you might have thought we d get a br...</td>\n",
       "      <td>News</td>\n",
       "      <td>December 21, 2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0   Donald Trump Sends Out Embarrassing New Year’...   \n",
       "1   Drunk Bragging Trump Staffer Started Russian ...   \n",
       "2   Sheriff David Clarke Becomes An Internet Joke...   \n",
       "3   Trump Is So Obsessed He Even Has Obama’s Name...   \n",
       "4   Pope Francis Just Called Out Donald Trump Dur...   \n",
       "5   Racist Alabama Cops Brutalize Black Boy While...   \n",
       "6   Fresh Off The Golf Course, Trump Lashes Out A...   \n",
       "7   Trump Said Some INSANELY Racist Stuff Inside ...   \n",
       "8   Former CIA Director Slams Trump Over UN Bully...   \n",
       "9   WATCH: Brand-New Pro-Trump Ad Features So Muc...   \n",
       "\n",
       "                                                text subject  \\\n",
       "0  Donald Trump just couldn t wish all Americans ...    News   \n",
       "1  House Intelligence Committee Chairman Devin Nu...    News   \n",
       "2  On Friday, it was revealed that former Milwauk...    News   \n",
       "3  On Christmas day, Donald Trump announced that ...    News   \n",
       "4  Pope Francis used his annual Christmas Day mes...    News   \n",
       "5  The number of cases of cops brutalizing and ki...    News   \n",
       "6  Donald Trump spent a good portion of his day a...    News   \n",
       "7  In the wake of yet another court decision that...    News   \n",
       "8  Many people have raised the alarm regarding th...    News   \n",
       "9  Just when you might have thought we d get a br...    News   \n",
       "\n",
       "                date  \n",
       "0  December 31, 2017  \n",
       "1  December 31, 2017  \n",
       "2  December 30, 2017  \n",
       "3  December 29, 2017  \n",
       "4  December 25, 2017  \n",
       "5  December 25, 2017  \n",
       "6  December 23, 2017  \n",
       "7  December 23, 2017  \n",
       "8  December 22, 2017  \n",
       "9  December 21, 2017  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspecting fake data:\n",
    "fake_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T15:24:57.397471Z",
     "iopub.status.busy": "2021-12-13T15:24:57.397332Z",
     "iopub.status.idle": "2021-12-13T15:24:57.571080Z",
     "shell.execute_reply": "2021-12-13T15:24:57.570351Z",
     "shell.execute_reply.started": "2021-12-13T15:24:57.397456Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>subject</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>As U.S. budget fight looms, Republicans flip t...</td>\n",
       "      <td>WASHINGTON (Reuters) - The head of a conservat...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 31, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>U.S. military to accept transgender recruits o...</td>\n",
       "      <td>WASHINGTON (Reuters) - Transgender people will...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 29, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Senior U.S. Republican senator: 'Let Mr. Muell...</td>\n",
       "      <td>WASHINGTON (Reuters) - The special counsel inv...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 31, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FBI Russia probe helped by Australian diplomat...</td>\n",
       "      <td>WASHINGTON (Reuters) - Trump campaign adviser ...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 30, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Trump wants Postal Service to charge 'much mor...</td>\n",
       "      <td>SEATTLE/WASHINGTON (Reuters) - President Donal...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 29, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>White House, Congress prepare for talks on spe...</td>\n",
       "      <td>WEST PALM BEACH, Fla./WASHINGTON (Reuters) - T...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 29, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Trump says Russia probe will be fair, but time...</td>\n",
       "      <td>WEST PALM BEACH, Fla (Reuters) - President Don...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 29, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Factbox: Trump on Twitter (Dec 29) - Approval ...</td>\n",
       "      <td>The following statements were posted to the ve...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 29, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Trump on Twitter (Dec 28) - Global Warming</td>\n",
       "      <td>The following statements were posted to the ve...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 29, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Alabama official to certify Senator-elect Jone...</td>\n",
       "      <td>WASHINGTON (Reuters) - Alabama Secretary of St...</td>\n",
       "      <td>politicsNews</td>\n",
       "      <td>December 28, 2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  As U.S. budget fight looms, Republicans flip t...   \n",
       "1  U.S. military to accept transgender recruits o...   \n",
       "2  Senior U.S. Republican senator: 'Let Mr. Muell...   \n",
       "3  FBI Russia probe helped by Australian diplomat...   \n",
       "4  Trump wants Postal Service to charge 'much mor...   \n",
       "5  White House, Congress prepare for talks on spe...   \n",
       "6  Trump says Russia probe will be fair, but time...   \n",
       "7  Factbox: Trump on Twitter (Dec 29) - Approval ...   \n",
       "8         Trump on Twitter (Dec 28) - Global Warming   \n",
       "9  Alabama official to certify Senator-elect Jone...   \n",
       "\n",
       "                                                text       subject  \\\n",
       "0  WASHINGTON (Reuters) - The head of a conservat...  politicsNews   \n",
       "1  WASHINGTON (Reuters) - Transgender people will...  politicsNews   \n",
       "2  WASHINGTON (Reuters) - The special counsel inv...  politicsNews   \n",
       "3  WASHINGTON (Reuters) - Trump campaign adviser ...  politicsNews   \n",
       "4  SEATTLE/WASHINGTON (Reuters) - President Donal...  politicsNews   \n",
       "5  WEST PALM BEACH, Fla./WASHINGTON (Reuters) - T...  politicsNews   \n",
       "6  WEST PALM BEACH, Fla (Reuters) - President Don...  politicsNews   \n",
       "7  The following statements were posted to the ve...  politicsNews   \n",
       "8  The following statements were posted to the ve...  politicsNews   \n",
       "9  WASHINGTON (Reuters) - Alabama Secretary of St...  politicsNews   \n",
       "\n",
       "                 date  \n",
       "0  December 31, 2017   \n",
       "1  December 29, 2017   \n",
       "2  December 31, 2017   \n",
       "3  December 30, 2017   \n",
       "4  December 29, 2017   \n",
       "5  December 29, 2017   \n",
       "6  December 29, 2017   \n",
       "7  December 29, 2017   \n",
       "8  December 29, 2017   \n",
       "9  December 28, 2017   "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspecting true data:\n",
    "true_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T15:24:57.594455Z",
     "iopub.status.busy": "2021-12-13T15:24:57.594322Z",
     "iopub.status.idle": "2021-12-13T15:24:57.749084Z",
     "shell.execute_reply": "2021-12-13T15:24:57.748059Z",
     "shell.execute_reply.started": "2021-12-13T15:24:57.594441Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Adding category-labels to each dataset: \n",
    "fake_df[\"label\"]=\"fake\"\n",
    "true_df[\"label\"]=\"true\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Merge fake- and true news into a single dataframe:\n",
    "merged_df = pd.concat([true_df, fake_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assessing whether merge was succesful:\n",
    "len(true_df) + len(fake_df) == len(merged_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Data cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Removing bad rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Remove rows with only whitespace and replace it with NaN:\n",
    "merged_df.replace(\" \", float(\"NaN\"), inplace=True)\n",
    "\n",
    "# Remove NA's:\n",
    "merged_df.dropna(subset = [\"text\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Remove duplicate texts:\n",
    "merged_df = merged_df.drop_duplicates(subset=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Reset indices:\n",
    "merged_df = merged_df.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Selecting only relevant columns:\n",
    "merged_df = merged_df[[\"text\", \"label\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Regex"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### Remove \"[city name] Reuters - \" from true articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\".*\\(Reuters\\) - \"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### Remove hashtags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"#(\\S+)\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### Remove twitter tags (\"@[username]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"@(\\S+)\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### Remove '(CAPSLOCK)' e.g. from (VIDEO); something which was quite frequent in the fake news dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"\\([A-Z]*\\)\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### Remove systematic patterns:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"The following statement.*accuracy[.]\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"pic\\.twitter\\.com\\/.* \"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define function:\n",
    "def remove_punctuation(text):\n",
    "    no_punct=[words for words in text if words not in string.punctuation]\n",
    "    words_wo_punct=''.join(no_punct)\n",
    "    return words_wo_punct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_df['text']=merged_df['text'].apply(lambda x: remove_punctuation(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Tokenization + Lower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define function:\n",
    "def tokenize(text):\n",
    "    split=re.split(\"\\W+\",text) \n",
    "    return split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_df['tokenized']=merged_df['text'].apply(lambda x: tokenize(x.lower()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define function:\n",
    "def remove_stopwords(text):\n",
    "    text=[words for words in text if words not in stopword]\n",
    "    #text=' '.join(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_df['tokenized'] = merged_df['tokenized'].apply(lambda x: remove_stopwords(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Lemmatize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define function:\n",
    "def penn2morphy(penntag):\n",
    "    morphy_tag = {'NN':'n', 'JJ':'a',\n",
    "                  'VB':'v', 'RB':'r'}\n",
    "    try:\n",
    "        return morphy_tag[penntag[:2]]\n",
    "    except:\n",
    "        return 'n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for i in range(len(merged_df['tokenized'])):\n",
    "    tagged = pos_tag(merged_df['tokenized'][i])\n",
    "    merged_df['tokenized'][i] = [lemmatizer.lemmatize(word, pos=penn2morphy(tag)) for word, tag in tagged]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Concatenate tokens into sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define function:\n",
    "def concat(text):\n",
    "    text=[words for words in text]\n",
    "    text=' '.join(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_df['text'] = merged_df['tokenized'].apply(lambda x: concat(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove newly induced empty columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_df.replace(\" \", float(\"NaN\"), inplace=True)\n",
    "\n",
    "merged_df.dropna(subset = [\"text\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_df = merged_df.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Assess whether we have missed anything"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "true_idx = merged_df[merged_df['label']==\"true\"].index.tolist()\n",
    "fake_idx = merged_df[merged_df['label']==\"fake\"].index.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('say', 113426),\n",
       " ('trump', 53621),\n",
       " ('u', 40552),\n",
       " ('state', 36143),\n",
       " ('would', 31145),\n",
       " ('president', 26582),\n",
       " ('republican', 20154),\n",
       " ('government', 19171),\n",
       " ('year', 18520),\n",
       " ('house', 16787)]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(\" \".join(merged_df['text'][true_idx]).split()).most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('trump', 58413),\n",
       " ('say', 36515),\n",
       " ('people', 19204),\n",
       " ('president', 18091),\n",
       " ('go', 17802),\n",
       " ('would', 17078),\n",
       " ('make', 16956),\n",
       " ('one', 16919),\n",
       " ('state', 16195),\n",
       " ('get', 14812)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(\" \".join(merged_df['text'][fake_idx]).split()).most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove newly found systematic patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"21st century wire say\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"21st century wire\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"filessupport.*\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"21wire\"\n",
    "\n",
    "for i in range(len(merged_df['text'])):\n",
    "    merged_df['text'][i] = re.sub(pattern, '', merged_df['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('trump', 58413),\n",
       " ('say', 36151),\n",
       " ('people', 19204),\n",
       " ('president', 18091),\n",
       " ('go', 17802),\n",
       " ('would', 17078),\n",
       " ('make', 16956),\n",
       " ('one', 16919),\n",
       " ('state', 16195),\n",
       " ('get', 14812)]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(\" \".join(merged_df['text'][fake_idx]).split()).most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Saving and loading cleaned dataset 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Write dataframe to csv-file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Selecting only relevant columns\n",
    "merged_df = merged_df[[\"text\", \"label\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Write to csv\n",
    "merged_df.to_csv(os.path.join(\"data\", \"generated_data\", \"cleaned_dataset_1.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# BERT trained- and evaluated on dataset 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Load cleaned data and prepare for classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "cleaned_dataset_1 = pd.read_csv(os.path.join(\"data\", \"generated_data\", \"cleaned_dataset_1.csv\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "__One row is corrupted when loading CSV and is turned into blank space. This is removed__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "cleaned_dataset_1.replace(\" \", float(\"NaN\"), inplace=True)\n",
    "\n",
    "cleaned_dataset_1.dropna(subset = [\"text\"], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "__Create training-, validiation and testing dataset:__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Create train/test split with 20% of all articles in testing data:\n",
    "train_1, test_1 = train_test_split(cleaned_dataset_1, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Create train/val split with 10% of remaining articles in validation data:\n",
    "train_1, val_1 = train_test_split(train_1, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assess that split was successful:\n",
    "len(train_1) + len(val_1) + len(test_1) == len(cleaned_dataset_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Convert label column to binary integer (0 = true, 1 = fake):\n",
    "train_1[\"label\"] = np.where(train_1[\"label\"] == \"true\", 0,1)\n",
    "val_1[\"label\"] = np.where(val_1[\"label\"] == \"true\", 0,1)\n",
    "test_1[\"label\"] = np.where(test_1[\"label\"] == \"true\", 0,1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8490</th>\n",
       "      <td>billionaire investor wilbur ross stand behind ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2952</th>\n",
       "      <td>white water raft bali visit temple java former...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24959</th>\n",
       "      <td>remember donald trump claim ford move job prod...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7294</th>\n",
       "      <td>u president barack obama foreign policy legacy...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23977</th>\n",
       "      <td>allege president donald trump create worldwide...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15028</th>\n",
       "      <td>saudi arabia announce confiscate money asset h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26379</th>\n",
       "      <td>go folk yet another video surface show yet ano...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21859</th>\n",
       "      <td>first impaneled grand jury grand jury begin is...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5890</th>\n",
       "      <td>iraqi say life danger work u government iraq f...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24288</th>\n",
       "      <td>republican long four year think progressive li...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label\n",
       "8490   billionaire investor wilbur ross stand behind ...      0\n",
       "2952   white water raft bali visit temple java former...      0\n",
       "24959  remember donald trump claim ford move job prod...      1\n",
       "7294   u president barack obama foreign policy legacy...      0\n",
       "23977  allege president donald trump create worldwide...      1\n",
       "15028  saudi arabia announce confiscate money asset h...      0\n",
       "26379  go folk yet another video surface show yet ano...      1\n",
       "21859  first impaneled grand jury grand jury begin is...      1\n",
       "5890   iraqi say life danger work u government iraq f...      0\n",
       "24288  republican long four year think progressive li...      1"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspecting transformed training data:\n",
    "train_1.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7660</th>\n",
       "      <td>trail opinion poll republican presidential nom...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30191</th>\n",
       "      <td>young man record maliciously abuse young woman...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8830</th>\n",
       "      <td>u house majority leader kevin mccarthy say tue...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19380</th>\n",
       "      <td>least 10 people die another 92 miss eastern de...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14784</th>\n",
       "      <td>medium outlet cnn radio free europe deutsche w...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38499</th>\n",
       "      <td>episode sunday wire show resume sunday may 29 ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33163</th>\n",
       "      <td>judge jeanine pirro leave disrespectful attitu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28982</th>\n",
       "      <td>bad enough oscar year black people nominate in...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32723</th>\n",
       "      <td>national security council aide craig deare dis...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28267</th>\n",
       "      <td>convenient stereotype girl small mishmash suga...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label\n",
       "7660   trail opinion poll republican presidential nom...      0\n",
       "30191  young man record maliciously abuse young woman...      1\n",
       "8830   u house majority leader kevin mccarthy say tue...      0\n",
       "19380  least 10 people die another 92 miss eastern de...      0\n",
       "14784  medium outlet cnn radio free europe deutsche w...      0\n",
       "38499  episode sunday wire show resume sunday may 29 ...      1\n",
       "33163  judge jeanine pirro leave disrespectful attitu...      1\n",
       "28982  bad enough oscar year black people nominate in...      1\n",
       "32723  national security council aide craig deare dis...      1\n",
       "28267  convenient stereotype girl small mishmash suga...      1"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspecting transformed validation data:\n",
    "val_1.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12510</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        text\n",
       "label       \n",
       "0      15312\n",
       "1      12510"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assess that data is roughly balanced across categories:\n",
    "train_1.groupby('label').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       text\n",
       "label      \n",
       "0      1672\n",
       "1      1420"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assess that data is rpughly balanced across categories:\n",
    "val_1.groupby('label').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3522</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       text\n",
       "label      \n",
       "0      4207\n",
       "1      3522"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assess that data is roughly balanced across categories:\n",
    "test_1.groupby('label').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define number of unique labels:\n",
    "n_labels = len(train_1['label'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Create list of texts to predict:\n",
    "X_dataset_1 = test_1['text'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7729"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect length\n",
    "len(X_dataset_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T15:25:09.651658Z",
     "iopub.status.busy": "2021-12-13T15:25:09.651049Z",
     "iopub.status.idle": "2021-12-13T15:51:54.972953Z",
     "shell.execute_reply": "2021-12-13T15:51:54.971887Z",
     "shell.execute_reply.started": "2021-12-13T15:25:09.651607Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/opt/conda/lib/python3.7/site-packages/simpletransformers/classification/classification_model.py:586: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
      "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bb5ed19312a4492ab028099b185de2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/27822 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f57b78d901846d1bec9a5d83f4a08f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ce5fac50b054090a606edb2ac5aa68d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 0 of 3:   0%|          | 0/218 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d2581f60f0c4d17bb7ff7ccfb494335",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 1 of 3:   0%|          | 0/218 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27efb8fafdf943b494c1bfb61fc9dd31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 2 of 3:   0%|          | 0/218 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(654, 0.06579963080759448)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize the model with the specified hyperparameters:\n",
    "FN_model_1 = ClassificationModel('bert',\"bert-base-uncased\",\n",
    "                                 num_labels=n_labels, use_cuda=False,\n",
    "                                 args={'reprocess_input_data': True, 'overwrite_output_dir': True,\n",
    "                                       \"num_train_epochs\": 3, \"max_seq_length\": 512, \"train_batch_size\": 128,\n",
    "                                       \"learning_rate\": 1e-5})\n",
    "\n",
    "# Fine-tune the model:\n",
    "FN_model_1.train_model(train_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "## Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Loading trained model, so we don't have to rerun the training each time we restart the kernel:\n",
    "FN_model_1 = ClassificationModel(\"bert\", \"outputs_dataset_1/\", num_labels=n_labels, use_cuda=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T16:01:14.489823Z",
     "iopub.status.busy": "2021-12-13T16:01:14.489294Z",
     "iopub.status.idle": "2021-12-13T16:09:26.815893Z",
     "shell.execute_reply": "2021-12-13T16:09:26.814067Z",
     "shell.execute_reply.started": "2021-12-13T16:01:14.489768Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f639493b6dda4c79983fbd3936f78f88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7729 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5facc2d1fafe4965a458da06038576ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/967 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the fine-tuned model to predict the testing labels and save the raw model outputs:\n",
    "_, raw_pred = FN_model_1.predict(X_dataset_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T16:18:41.408426Z",
     "iopub.status.busy": "2021-12-13T16:18:41.407736Z",
     "iopub.status.idle": "2021-12-13T16:18:41.414977Z",
     "shell.execute_reply": "2021-12-13T16:18:41.414015Z",
     "shell.execute_reply.started": "2021-12-13T16:18:41.408373Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert raw model outputs to class probabilities:\n",
    "probabilities = softmax(raw_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T16:18:42.555636Z",
     "iopub.status.busy": "2021-12-13T16:18:42.555058Z",
     "iopub.status.idle": "2021-12-13T16:18:42.564844Z",
     "shell.execute_reply": "2021-12-13T16:18:42.564204Z",
     "shell.execute_reply.started": "2021-12-13T16:18:42.555588Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4.79282272e-04, 9.99520718e-01],\n",
       "       [9.99563380e-01, 4.36620433e-04],\n",
       "       [4.79260344e-04, 9.99520740e-01],\n",
       "       ...,\n",
       "       [3.38563774e-03, 9.96614362e-01],\n",
       "       [5.35239229e-04, 9.99464761e-01],\n",
       "       [9.99326263e-01, 6.73736558e-04]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Asssess probabilities:\n",
    "probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T16:18:43.449921Z",
     "iopub.status.busy": "2021-12-13T16:18:43.449067Z",
     "iopub.status.idle": "2021-12-13T16:18:43.465165Z",
     "shell.execute_reply": "2021-12-13T16:18:43.464636Z",
     "shell.execute_reply.started": "2021-12-13T16:18:43.449872Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Binarize probabilities to the most probable class:\n",
    "binary_preds = [np.argmax(pred) for pred in probabilities] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T16:18:46.642016Z",
     "iopub.status.busy": "2021-12-13T16:18:46.641499Z",
     "iopub.status.idle": "2021-12-13T16:18:46.650470Z",
     "shell.execute_reply": "2021-12-13T16:18:46.649422Z",
     "shell.execute_reply.started": "2021-12-13T16:18:46.641965Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7729"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect length of predictions:\n",
    "len(binary_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "Collapsed": "false",
    "execution": {
     "iopub.execute_input": "2021-12-13T16:18:47.307880Z",
     "iopub.status.busy": "2021-12-13T16:18:47.307401Z",
     "iopub.status.idle": "2021-12-13T16:18:47.329966Z",
     "shell.execute_reply": "2021-12-13T16:18:47.329346Z",
     "shell.execute_reply.started": "2021-12-13T16:18:47.307831Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      4187\n",
      "           1       1.00      1.00      1.00      3542\n",
      "\n",
      "    accuracy                           1.00      7729\n",
      "   macro avg       1.00      1.00      1.00      7729\n",
      "weighted avg       1.00      1.00      1.00      7729\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[4182,    5],\n",
       "       [  13, 3529]])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print classification report:\n",
    "print(classification_report(test_1.label, binary_preds))\n",
    "\n",
    "# Print confusion matrix:\n",
    "confusion_matrix(test_1.label, binary_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Preprocess dataset 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Data loading and wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "file_list = glob.glob(os.path.join(os.getcwd(), \"data\", \"dataset_2\", \"fake\", \"*.txt\"))\n",
    "\n",
    "fake = []\n",
    "\n",
    "for file_path in file_list:\n",
    "    with open(file_path, encoding='windows-1252') as f_input:\n",
    "        encoded_f = f_input.read().replace(\"\\n\", \" \")\n",
    "        fake.append(encoded_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "file_list = glob.glob(os.path.join(os.getcwd(), \"data\", \"dataset_2\", \"real\", \"*.txt\"))\n",
    "\n",
    "real = []\n",
    "\n",
    "for file_path in file_list:\n",
    "    with open(file_path, encoding='windows-1252') as f_input:\n",
    "        encoded_f = f_input.read().replace(\"\\n\", \" \")\n",
    "        real.append(encoded_f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Remove \\ from the data:\n",
    "for i in range(len(fake)):\n",
    "    fake[i] = fake[i].replace(\"\\'\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Remove \\ from the data:\n",
    "for i in range(len(real)):\n",
    "    real[i] = real[i].replace(\"\\'\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Convert data to pandas dataframe:\n",
    "fake_new = pd.DataFrame(fake)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Rename column with texts to text:\n",
    "fake_new = fake_new.rename({0: \"text\"},axis = 'columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Add label-column with fake labels:\n",
    "fake_new['label'] = 'fake'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Convert data to pandas dataframe:\n",
    "real_new = pd.DataFrame(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Rename column with texts to text:\n",
    "real_new = real_new.rename({0: \"text\"},axis = 'columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Add label-column with fake labels:\n",
    "real_new['label'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Merge fake- and true news into a single dataframe:\n",
    "merged_new = pd.concat([fake_new, real_new])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Reset indeces:\n",
    "merged_new = merged_new.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Selecting only relevant columns:\n",
    "merged_new = merged_new[[\"text\", \"label\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The warranty on ‘Make America Great Again’ bas...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Calling it a total disaster, president-elect D...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WASHINGTON, D.C. –   Former presidential inter...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>President Barack Obama’s legacy might soon be ...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>atican City – In a final speech to the synod, ...</td>\n",
       "      <td>fake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>WASHINGTON — Republicans are united on repeali...</td>\n",
       "      <td>true</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>President-elect Donald Trump escalated his rhe...</td>\n",
       "      <td>true</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>Congress is preparing to do major battle next ...</td>\n",
       "      <td>true</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>PALM BEACH, Fla. -- President-elect Donald Tru...</td>\n",
       "      <td>true</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>This is my last column until after the electio...</td>\n",
       "      <td>true</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>248 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text label\n",
       "0    The warranty on ‘Make America Great Again’ bas...  fake\n",
       "1    Calling it a total disaster, president-elect D...  fake\n",
       "2    WASHINGTON, D.C. –   Former presidential inter...  fake\n",
       "3    President Barack Obama’s legacy might soon be ...  fake\n",
       "4    atican City – In a final speech to the synod, ...  fake\n",
       "..                                                 ...   ...\n",
       "243  WASHINGTON — Republicans are united on repeali...  true\n",
       "244  President-elect Donald Trump escalated his rhe...  true\n",
       "245  Congress is preparing to do major battle next ...  true\n",
       "246  PALM BEACH, Fla. -- President-elect Donald Tru...  true\n",
       "247  This is my last column until after the electio...  true\n",
       "\n",
       "[248 rows x 2 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspecting:\n",
    "merged_new "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_new['text']=merged_new['text'].apply(lambda x: remove_punctuation(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Tokenize and lower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_new['tokenized']=merged_new['text'].apply(lambda x: tokenize(x.lower()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_new['tokenized'] = merged_new['tokenized'].apply(lambda x: remove_stopwords(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Lemma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for i in range(len(merged_new['tokenized'])):\n",
    "    tagged = pos_tag(merged_new['tokenized'][i])\n",
    "    merged_new['tokenized'][i] = [lemmatizer.lemmatize(word, pos=penn2morphy(tag)) for word, tag in tagged]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Concatenate tokens into sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "merged_new['text'] = merged_new['tokenized'].apply(lambda x: concat(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Write dataframe to csv-file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Selecting only relevant columns:\n",
    "merged_new = merged_new[[\"text\", \"label\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Write to csv:\n",
    "merged_new.to_csv(os.path.join(\"data\", \"generated_data\", \"cleaned_dataset_2.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# BERT trained on dataset 1, evaluated on dataset 2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Load cleaned data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "cleaned_dataset_2 = pd.read_csv(os.path.join(\"data\", \"generated_data\", \"cleaned_dataset_2.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Change labels to binary integers:\n",
    "cleaned_dataset_2[\"label\"] = np.where(cleaned_dataset_2[\"label\"] == \"true\", 0,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define number of unique labels:\n",
    "n_labels = len(cleaned_dataset_2['label'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Prepare and predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06a27bcc526548af9193ac4523c2fd9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/248 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61d53df95a4c49ee9c757f69b06ad13e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/31 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the 1st fine-tuned model to predict dataset 2 save the raw model outputs:\n",
    "_, raw_pred = FN_model_1.predict(cleaned_dataset_2['text'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Convert raw model outputs to class probabilities:\n",
    "probabilities = softmax(raw_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Binarize probabilities to the most probable class:\n",
    "binary_preds = [np.argmax(pred) for pred in probabilities] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "248"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect length of predictions:\n",
    "len(binary_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      0.62      0.61       128\n",
      "           1       0.57      0.55      0.56       120\n",
      "\n",
      "    accuracy                           0.58       248\n",
      "   macro avg       0.58      0.58      0.58       248\n",
      "weighted avg       0.58      0.58      0.58       248\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[79, 49],\n",
       "       [54, 66]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print classification report:\n",
    "print(classification_report(cleaned_dataset_2.label, binary_preds))\n",
    "\n",
    "# Print confusion matrix:\n",
    "confusion_matrix(cleaned_dataset_2.label, binary_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# BERT trained- and evaluated dataset 2:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Create train/test split with 20% of all articles in testing data:\n",
    "train_2, test_2 = train_test_split(cleaned_dataset_2, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Create list of texts to predict:\n",
    "X_dataset_2 = test_2['text'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Define number of unique labels:\n",
    "n_labels = len(train_2['label'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/opt/conda/lib/python3.7/site-packages/simpletransformers/classification/classification_model.py:586: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
      "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a91196a98ad6415fba3e4470d1b2f71f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/198 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a524dae887384d85b1667c91dc1097e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/15 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71036e9c301f493ebc08888cfa31578c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 0 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "848fed44b023480d9b3bd4a2ee170e2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 1 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "623540cf55224179b973d505a4bf6b17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 2 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d40e87d593ec4733b2dddab47b48bb4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 3 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4f96873bfc942df80275f8916d7d37c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 4 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b654564844f467bad43d0aadec441c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 5 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8dcebc9564e4ec3aa7f5cdfbebab02e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 6 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2829371ac35c473abb74c8c0175e8ad9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 7 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac800c43351e48a39621815181593248",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 8 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00bc6240e82846c3b68c8d0330a54e57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 9 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad9ade127e274e5f9f2c0a96965d8bd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 10 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9c0590bee6d4bfa912688fab6c89565",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 11 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96e2e807ac5b42cf9f6edd4a3ff2f3fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 12 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51caec65ab72435b9bcb71c1790c5941",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 13 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3724cd83623e48c88598a376c3737298",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 14 of 15:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(195, 0.3403184807071319)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize the model with the specified hyperparameters:\n",
    "FN_model_2 = ClassificationModel('bert',\"bert-base-uncased\",\n",
    "                                 num_labels=n_labels, use_cuda=False,\n",
    "                                 args={'reprocess_input_data': True, 'overwrite_output_dir': True,\n",
    "                                       \"num_train_epochs\": 15, \"max_seq_length\": 512, \"train_batch_size\": 16,\n",
    "                                       \"learning_rate\": 1e-5})\n",
    "\n",
    "# Fine-tune the model:\n",
    "FN_model_2.train_model(train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Loading trained model, so we don't have to rerun the training each time we restart the kernel:\n",
    "FN_model_2 = ClassificationModel(\"bert\", \"outputs_dataset_2/\", num_labels=n_labels, use_cuda=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f27181b8ac93484080ccbca223c25d43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/50 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd37923895a5414b87afabd09bb3404f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the fine-tuned model to predict the testing labels and save the raw model outputs:\n",
    "_, raw_pred = FN_model_2.predict(X_dataset_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Convert raw model outputs to class probabilities:\n",
    "probabilities = softmax(raw_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Binarize probabilities to the most probable class:\n",
    "binary_preds = [np.argmax(pred) for pred in probabilities] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect length of predictions:\n",
    "len(binary_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.88      0.82        26\n",
      "           1       0.85      0.71      0.77        24\n",
      "\n",
      "    accuracy                           0.80        50\n",
      "   macro avg       0.81      0.80      0.80        50\n",
      "weighted avg       0.81      0.80      0.80        50\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[23,  3],\n",
       "       [ 7, 17]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print classification report:\n",
    "print(classification_report(test_2.label, binary_preds))\n",
    "\n",
    "# Print confusion matrix:\n",
    "confusion_matrix(test_2.label, binary_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# BERT trained on dataset 2, evaluated on dataset 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e65bc027d5c84c308a902cb398dcb896",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/38643 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b02a2c9876bd461c8a8fb769236416a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4831 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the fine-tuned model to predict the texts from dataset 1 and save the raw model outputs:\n",
    "_, raw_pred = FN_model_2.predict(cleaned_dataset_1['text'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Convert raw model outputs to class probabilities:\n",
    "probabilities = softmax(raw_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Binarize probabilities to the most probable class:\n",
    "binary_preds = [np.argmax(pred) for pred in probabilities] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38643"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect length of predictions:\n",
    "len(binary_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "cleaned_dataset_1[\"label\"] = np.where(cleaned_dataset_1[\"label\"] == \"true\", 0,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.66      0.51      0.58     21191\n",
      "           1       0.54      0.69      0.60     17452\n",
      "\n",
      "    accuracy                           0.59     38643\n",
      "   macro avg       0.60      0.60      0.59     38643\n",
      "weighted avg       0.61      0.59      0.59     38643\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[10848, 10343],\n",
       "       [ 5466, 11986]])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print classification report:\n",
    "print(classification_report(cleaned_dataset_1.label, binary_preds))\n",
    "\n",
    "# Print confusion matrix:\n",
    "confusion_matrix(cleaned_dataset_1.label, binary_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Periods - for temporal word embedding analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Data wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import regex as re\n",
    "from datetime import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Load data:\n",
    "fake = pd.read_csv(os.path.join(\"data\", \"dataset_1\", \"Fake.csv\"))\n",
    "\n",
    "# NA for wrong entries:\n",
    "fake[\"date\"] = [re.sub(\"^.*:.*|^.* .* .* .*|^\\d.*\", \"NA\", date) for date in fake[\"date\"]] # All webpages, entries that start with a number and sequences of words upon words upon words should be NA\n",
    "\n",
    "# Drop rows with NAs:\n",
    "fake = fake[(fake!='NA').all(1)]\n",
    "\n",
    "# Streamline dates:\n",
    "months = [\"January\", \"February\", \"March\", \"April\", \"May\", \"June\", \"July\", \"August\", \"September\", \"October\", \"November\", \"December\"]\n",
    "for i in months:\n",
    "    fake[\"date\"] = [re.sub(f\"^{i}\", f\"{i[0:3]}\", date) for date in fake[\"date\"]]\n",
    "\n",
    "# Convert to date format:\n",
    "fake[\"date\"] = [datetime.strptime(date, \"%b %d, %Y\").date() for date in fake[\"date\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Find date range:\n",
    "date_range = max(fake[\"date\"]) - min(fake[\"date\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Create categorical variable pertaining to split:\n",
    "period = []\n",
    "for date in fake[\"date\"]:\n",
    "    if date <= min(fake[\"date\"]) + date_range/5:\n",
    "        period.append(1)\n",
    "    if date > min(fake[\"date\"]) + date_range/5 and date <= min(fake[\"date\"]) + date_range/5*2:\n",
    "        period.append(2)\n",
    "    if date > min(fake[\"date\"]) + date_range/5*2 and date <= min(fake[\"date\"]) + date_range/5*3:\n",
    "        period.append(3)\n",
    "    if date > min(fake[\"date\"]) + date_range/5*3 and date <= min(fake[\"date\"]) + date_range/5*4:\n",
    "        period.append(4)\n",
    "    if date > min(fake[\"date\"]) + date_range/5*4:\n",
    "        period.append(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Create column with periods:\n",
    "fake[\"period\"] = period"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 4, 3, 2, 1])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ensure that the unique entries in the period-column is correct:\n",
    "fake['period'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Write data to csv:\n",
    "fake.to_csv(os.path.join(\"data\", \"generated_data\", \"fake_periods.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Load data from csv:\n",
    "fake = pd.read_csv(os.path.join(\"data\", \"generated_data\", \"fake_periods.csv\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Remove rows with only whitespace and replace it with NA's\n",
    "fake.replace(\" \", float(\"NaN\"), inplace=True)\n",
    "\n",
    "# Remove NA's\n",
    "fake.dropna(subset = [\"text\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Remove duplicate texts:\n",
    "fake = fake.drop_duplicates(subset=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Reset indeces:\n",
    "fake = fake.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\".*\\(Reuters\\) - \"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove hashtags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"#(\\S+)\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"@(\\S+)\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove (capslock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"\\([A-Z]*\\)\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove systematic patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"The following statement.*accuracy[.]\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"pic\\.twitter\\.com\\/.* \"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove punctuation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fake['text']=fake['text'].apply(lambda x: remove_punctuation(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Tokenize and lower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fake['tokenized']=fake['text'].apply(lambda x: tokenize(x.lower()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fake['tokenized'] = fake['tokenized'].apply(lambda x: remove_stopwords(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Lemmatize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(fake['tokenized'])):\n",
    "    tagged = pos_tag(fake['tokenized'][i])\n",
    "    fake['tokenized'][i] = [lemmatizer.lemmatize(word, pos=penn2morphy(tag)) for word, tag in tagged]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Concatenate words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fake['text'] = fake['tokenized'].apply(lambda x: concat(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove newly induced empty columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fake.replace(\" \", float(\"NaN\"), inplace=True)\n",
    "\n",
    "fake.dropna(subset = [\"text\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fake = fake.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove more systematic patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"21st century wire say\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"21st century wire\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"filessupport.*\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# Define regex pattern:\n",
    "pattern = r\"21wire\"\n",
    "\n",
    "for i in range(len(fake['text'])):\n",
    "    fake['text'][i] = re.sub(pattern, '', fake['text'][i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Write data for word-embedding analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Concatenate\n",
    "period_texts = []\n",
    "for i in range(1, 6):\n",
    "    period_text = \" \".join(fake.loc[fake['period'] == i][\"text\"])\n",
    "    period_texts.append(period_text)\n",
    "    \n",
    "# Write as .txt files\n",
    "for i, n in zip(period_texts, range(1,6)):\n",
    "    text_file = open(os.path.join(\"word_embeddings\", \"output\", \"texts\", f\"00{n}0.txt\"), \"w\")\n",
    "    n = text_file.write(i)\n",
    "    text_file.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}