02_clean_data_backup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'slate3k'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ab044b27d7c6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mscraper_functions\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mhas_no_words\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcorpus\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mstopwords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mstopwords\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/duterte-speech-analysis/scraper_functions.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0murllib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0murlparse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mslate3k\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mslate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPyPDF2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtika\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'slate3k'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from scraper_functions import has_no_words\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "stopwords = stopwords.words('english')\n",
    "stopwords.remove('no')\n",
    "stopwords.remove('not')\n",
    "stopwords.append('sa')\n",
    "stopwords.append('na')\n",
    "stopwords.append('ng')\n",
    "stopwords.append('ko')\n",
    "stopwords.append('ba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(421, 3)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('pypdf_transcript.csv', index_col=0)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-04-27</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-04-24</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-04-16</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Intervention of President Rodrigo Roa Duterte ...</td>\n",
       "      <td>2020-04-14</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Intervention of President Rodrigo Roa Duterte ...</td>\n",
       "      <td>2020-04-14</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-04-13</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>Meeting of President Rodrigo Roa Duterte with ...</td>\n",
       "      <td>2020-04-08</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>Talk to the Nation of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-04-06</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Talk to the Nation on COVID-19 of President Ro...</td>\n",
       "      <td>2020-04-03</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>Nation Address of President Rodrigo Roa Dutert...</td>\n",
       "      <td>2020-04-01</td>\n",
       "      <td>Presidential Communications Operations Office\\...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title       date  \\\n",
       "0  Talk to the People of President Rodrigo Roa Du... 2020-04-27   \n",
       "1  Talk to the People of President Rodrigo Roa Du... 2020-04-24   \n",
       "2  Talk to the People of President Rodrigo Roa Du... 2020-04-16   \n",
       "3  Intervention of President Rodrigo Roa Duterte ... 2020-04-14   \n",
       "4  Intervention of President Rodrigo Roa Duterte ... 2020-04-14   \n",
       "5  Talk to the People of President Rodrigo Roa Du... 2020-04-13   \n",
       "6  Meeting of President Rodrigo Roa Duterte with ... 2020-04-08   \n",
       "7  Talk to the Nation of President Rodrigo Roa Du... 2020-04-06   \n",
       "8  Talk to the Nation on COVID-19 of President Ro... 2020-04-03   \n",
       "9  Nation Address of President Rodrigo Roa Dutert... 2020-04-01   \n",
       "\n",
       "                                                text  \n",
       "0  Presidential Communications Operations Office\\...  \n",
       "1  Presidential Communications Operations Office\\...  \n",
       "2  Presidential Communications Operations Office\\...  \n",
       "3  Presidential Communications Operations Office\\...  \n",
       "4  Presidential Communications Operations Office\\...  \n",
       "5  Presidential Communications Operations Office\\...  \n",
       "6  Presidential Communications Operations Office\\...  \n",
       "7  Presidential Communications Operations Office\\...  \n",
       "8  Presidential Communications Operations Office\\...  \n",
       "9  Presidential Communications Operations Office\\...  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['date'] = pd.to_datetime(df['date'], format='%B %d, %Y')\n",
    "df = df.sort_values('date', ascending=False).reset_index().drop('index', axis=1)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speech of President Rodrigo Roa Duterte during the Presidential Anti Corruption Commission (PACC) 1st Anniversary\n",
      "\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "Kindly sit down. Salamat po.\n",
      " \n",
      " \n",
      " \n",
      "Until today, so many billions are lost everyday because of corruption and I \n",
      "only have three pages to tell you what is it all about. It does not say \n",
      "anything so I will just\n",
      " \n",
      "[laughter]\n",
      " \n",
      "read the \n",
      "salutations and proceed on my \n",
      "own.\n",
      " \n",
      " \n",
      " \n",
      "The Executive Secretary Medialdea and other members of Cabinet; His \n",
      "Excellency Zhao Jianhua; officials and employees of the Office of the \n",
      "Presidential Anti\n",
      "-\n",
      "Corruption Commission; fellow workers in government; \n",
      "other dist\n",
      "inguished guests; ladies and gentlemen\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "Let me relate to you or narrate two events which happened during the last \n",
      "four mounts. The first was a\n",
      "===============================\n",
      "ong kalokohan.\n",
      " \n",
      "[applause]\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "Alam mo, nandiyan man sila lahat. They are \n",
      "all here. No transaction of \n",
      "\n",
      "reclamation na kung ano, approval of contracts, it does not reach my table. I \n",
      "have prohibited that.\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "Kayo \n",
      "---\n",
      " \n",
      "kaya nga ABS\n",
      "-\n",
      "CBN, sabihin ko, wala kayon\n",
      "g makalkal sa akin sa \n",
      "pagkatao ko basta pera. Because maski piso, the contracts, the documents \n",
      "begins and ends there sa Malaca \n",
      "---\n",
      " \n",
      "sa Cabinet member. Wala talaga ako.\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "So what I get is my salary, 204,000. But you know that \n",
      "---\n",
      " \n",
      "sabihin ninyo, \n",
      "\n",
      "\n",
      "because I have two families.\n",
      " \n",
      "[laughter]\n",
      " \n",
      "I have to support the grandmother \n",
      "\n",
      "-\n",
      "i\n",
      "-\n",
      "\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "\n",
      " \n",
      "I challenge you, \n",
      "your one peso will win one million sa akin. Wala kang makita sa Malacañan \n",
      "nagpirma ako ng pera either representation o ano, even clothing allowance \n",
      "wala ako.\n",
      " \n",
      " \n",
      " \n",
      "\n",
      "mapreso m\n",
      "an ako. It could not be corruption. That would be very \n",
      "impossible.\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "\n",
      "\n",
      "\n",
      " \n",
      "mine.\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "Maraming salamat po.\n",
      "  \n",
      "[applause]\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      "---\n",
      " \n",
      "END \n",
      "---\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Example of speech; remove everything before the date\n",
    "entry = df.loc[200]\n",
    "sample = entry['text']\n",
    "index = sample.find(']')\n",
    "print(entry['title'])\n",
    "print(sample[index+1:1000])\n",
    "print('===============================')\n",
    "print(sample[-1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(text):\n",
    "    index_of_date = text.find(']')\n",
    "    text = text[index_of_date+1:]\n",
    "    text = text.lower()\n",
    "#     I included the asterisk * to include swears\n",
    "    text = re.sub(r'([^A-Z a-z*\\'])|', '', text)\n",
    "    \n",
    "#     Not sure if I will still remove stopwords\n",
    "#     text = ' '.join([word for word in text.split() if word not in stopwords])\n",
    "\n",
    "    text = ' '.join(text.split())\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clean_text'] = df['text'].apply(clean_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"kindly sit down salamat po until today so many billions are lost everyday because of corruption and i only have three pages to tell you what is it all about it does not say anything so i will just laughter read the salutations and proceed on my own the executive secretary medialdea and other members of cabinet his excellency zhao jianhua officials and employees of the office of the presidential anticorruption commission fellow workers in government other distinguished guests ladies and gentlemen let me relate to you or narrate two events which happened during the last four mounts the first was a command conference of all armed forces and police officers on that day i was quite busy dismantling the board of and without any public bidding and as everybody knows i have always stressed bidding that is fair and just wala ho silang bidding and on top of that is the nayong pilipino and they contract with a gambling casino in that area and so i was not really feeling great when i arrived here for the in this room for the command conference i sonny dominguez passed many many a complaint about medicines purchases by the afp hospital for the soldiers and that there were some ghost deliveries in that file they gave me so i just lost my patience and i told the military and the police i am really very sad because kayo pa naman ang pinagkatiwalaan ko to cover my back and here you are this paying none for the soldiers sabi ko you you you can f*** with was really for the soldiers and so in my exasperation i said you know i am really very tired i think my mindset now cannot absorb so much for all of this and so i said i would like to offer to you my resignation if the four branches of the armed forces and the police albayalde was there if you would stand up all of you then that is an indication that you want another president then i will step down tonight and so nobody stood up and so we just i let it pass and well i said i a delivery which never came it concerned an office in the dar department of agrarian reforms and all my cabinet was there present and there was this complaint that some of those applications for conversion came in in almost two to three years so i told castriciones maggawaan mo ako ng simplified during the cabinet meeting because if i find it good then i can suggest the other departments to follow and when he started to show it though not his fault really i am not blaming him but the way the bureaucracy is set up it is really conducive to or for corruption to thr you would need a radical change overnight if you want corruption to stop so i proceeded about four pages full of requirements clearance from the nib clearance from waterworks clearance from the lgus nako patay na tapos clearance and clearance he was talking i stoop up and i said that documents that you are suggesting to make things faster is not for me it belongs to two about two presidents from now with those requirements you will ensure corruption and it would come naturally and automatic kung ganun karaming clearances magkuha ng clearance sa waterworks magkuha ka ng clearance sa practically in all ang wala doon ang foreign affairs lang laughter listening to you because that is not my concern it cannot work on me so i stood up and i said sinabi nandiyan sila roy sinabi ko sa kanila want exaltations or whatever i just want simply to resign and rest at pagtindig ko sinabi ko because most of my number of retired military officers and the civilians parang almost hati na tapos doon na ako sa labas well of course castriciones proceeded to rattle ahead with the executive secretary and maybe invite some guys from the agree on certain procedures kasi kung anoano na lang rin ang pinaglalagay kasi iyong regulations nila nagpatong doon sa they add more and more regulations and it dito nagit could not really be corruption but it could be shortsightedness or plain i will not follow that because that is stupid ganun eh regulation then you are hailed to the sandiganbayan for gross violation of regulations you know you are not congress it is only congress who can mete out imprisonments by law katok laughter they are lost in a maze of so many requirements some are really stupid who is this guy before the end of the year i will just suggest and if it is accepted by all fine then at the end of the year they would say that you have to you have to reimburse government eh p***** i** nagastos ko na saan ako maghanap sandigan kasi ito sila at noon if you want to buy computers on the local level you have to go to manila for permission i do not really see the rationale of these requirements but some stupid enterprising guys would just come up and to make money what for kasi pupunta pa kami dito get those requirements and you have to spend money following up considering the monster of traffic that you have to deal with now they wanted us to buy a computer that was something like a piece itong akin naman it was sold by a korean ganun rin puro made in ano mo so i did not so i proceeded but then on the other hand the lawyer and me says that well mukhang magkocomprise na lang siguro but i signed the contract pero nawithdraw ko alam mo idenemanda ako ng coa and the guy there sa sandi the ombudsman went back and forth and to think that he had the gall to ask me to pay for his hotel bills and all ikaw naman pagka gusto mo sila ang mga ano ko sabi ko ayaw ko kung magbayad ako sa hotel nila p***** i** papatayin ko laughter niyan there will be a crescendo of complaints pero but after that three well anyway i they it was filed with the sandigan i filed the motion to dismiss the garbled to evidence denied i went to the supreme court i was acquitted both on procedural and substantial issues kaya hindi nila alam i was i became president i experienced it all kaya huwag mo akong p***** i***** yayariin kita laughter hindi ako nagyayabang i was mayor for years i was a congressman for one term then i became iyan ang ikulong laughter then i went back to being nako diyos ko and the other one itong coa from every year nagpapaparty ako so i have a day during december for the blind people they have a night then the following night then we have the differently abled just to be look as long as the webster dictionary says it is disabled then they should be called disabled huwag na natin pagandahin iyang miserable diyan you bring to fore their situation and mahighlight tuloy ang arrest magpunta ng pupunta ng bar pupunta ng kung ano mga buang kasi laughter hindi ang condition that they are supervised by the city health officer icewomen na hindi if only if the city health officer would request for the police to intervene otherwise sabi ko sa kanila if you go to the city health officer you have this attendance card and you use condom fine with me sige you work the whole night if you enjoy it iyon lang walang walang vice squad vice squad eh in all of my years as a student of law and as a lawyer and as president i have yet to hear a single prostitute being prosecuted in court for exercising tapos igood time tapos binubugbog pa iyan ang ayaw ko so i decided altogether do not because if you do that may bugbog ka sa akin nangbganon time corruption ang problema nito every table diyan sa opisina ayaw kong magmention kasi lahat finofollow up ka basta clearance basta clearance or a certification from this office they simply sit on it at pabalik kaya ngayon ganito i have announced it then and now that anybody who has a complaint for itong mga corruption extortion pati itong big business you have hours to do it just knock at the door sa malacaan tapos bi ni duterte kausapin ko raw siya kunin mo ako lang ang ako lang ang presidente nila nakita nangbubog ewan ko dito kung hindi kita ihulog diyan sa pasiglaughter lahat ng extortionextortion pagdating diyan sabihin ko you like money you eat it para masanay and those are frustrations by the way those are the things that contribute to my dismay kaya mawawalaan ka mag gana ng trabaho and the big ones pero dito sa big ones because sonny nandiyan sila bingbong hindi ka talaga makakalusot iyong mga maliit lang pero dito sa itaas cannot walang shake down dito and most of my ilan kayo roy mga exmilitary men bakit gusto ko mag gusto kong magsafety so that the soldiers magpatayan but i got military men because i can have my way they know when they were assigned in mindanao nafrom davao like roy cimatu he once upon a time passed there as the commander of the th na dinala niya noon magkaibigan kami dahil magkabarkada kami gabigabi sa labas nagaano the camp can be a very lonely place so we became fast friends but i know ganito susmaryosep sabi ko sanay ako sanay ako i started sabi ko si esperon tapos nandiyan si delfin lorenzana and then sa denr bumitaw si ah hindi so from then on i got the all military magbakante ngayon military bakit sa military isang utos mo lang gagawin talaga and they know that i ha that i cover their backs and also they know because they happen to be assigned in my city that i will not order them something that is wrong utos denr man si roy sabi ko cretary ao here wala si secretary ao anong ginawa niya baka bumalik sa boracay maraming babae doon naliligo eh laughter pagsabi ng mga tao doon mag kaso kasi ganon eh alam mo ang mga gambling i like it especially if the winner is my friend may mabalato ka what i do not like about ghostage nila policemen there gambling s*** you can eat your years contract you will not gamble there magdemanda ka ng sinong idemanda magdemanda ka ah hindi ka makapunta doon sa boracay sabi ko well but what one funny thing may pagkain tayo dito magkain kami hindi wala tapos na mabuti pa sila nauna pa sa presidente laughter you know i the day i told roy nandoon na sa boracay kinabukasan lumabas ng philippine sunstar roy was walking along the beach of boracay there were two caucasians laughter ah ganun sila ut i think the angling of the camera was just too laughter siguro inutusan kitang laughter t*** i** years ganun sabi ko wala pollutant you close it na ako magano sa totoo ako and they are all honest men i know them because they serve davao and i would know wala naman police pa pero itong mga military trabaho lang and i think they are very thankful not to me but the opportunity to be of retire eh twenty rather bata pa lugi the government general katagalan niyan at the age of magretire lugi almost billion si kim henares she just assessed it at eight from billion to million mistah because i was adopted by the class nine ma guerrero ng customs maisahan sila eh ah i mean lapen lapea and faeldon too naive too trusting alam mo diyan sa customs kung nandiyan ka pasensya nalang pero that is the truth talagang puro palusotan diyan i si lapea so that nung napahiya kasi nalusotan eh got him sa ah itong tesda which is a very at si faeldon ganun rin ang nangyari ngayon may isa akong inilagay diyan si guerrero you might have bakit is there any solution kung you you you time and again lokohin mo kami kailangan ko ng army nilagyan ko ng mga coast guard sabi ko kung coast guard ang magloko ipabaril mo sa army kung army ang magloko ipabaril mo sa pulis ger si jagger hindi tumatawa pareho kay ao si jagger parang kunsumido laughter kunsumido palagi parang gustong pumatay kaya kasi gusto ninyo palliativepalliative sabi ko barilin mo sabi ko tutal garbled wala na cremate niyan no more ganun ang ano eh ganun ang pilipinas luslusin ka nang luslusin eh itong pulis well pulis rin pati pdea mauuna sa impiyerno hindi ako you kakapatay nila ilan na lang naiwan na ano puro matitino na eh minsan ano ni eleazar and he was criticized for that sabihin mo kay laughter ganun sumasayaw nga iyan sila na tao sabi mo i have his back covered kasi nung pagumpisa nitong drug war sinabi ko talaga sa kanya ito magprangkahan tayo ilan ang nakita ninyo sa previous administration na nagsurrender huli siguro one or two or three buy bust diyan malaki dito and then one time so nung napresidente ako daladala ko si bato bato is good he is now a candidate for senator sabihin ko sa inyo si bato and all others saan si morente is morente here all of doon siya and he has been with us pabalik assigned somewhere else then back to davao nung nagpresidente ako dala because i know that the drug problem is far worse than we ever thought at that time so pagda pagdating ko after two weeks pumunta siya dito daladala na niya there were about six generals of the police were involved ambush buhay pa rin laughter t**** i** ka y hindi t*** i** hindi ako nagpa hindi ako nagpaambush ha laughter ninyo same room dito pinaalis ko lahat nilock ko tapos sabi ko magprangkahan tayo i feel very corny lyour peer mayor ako eh i do not want to sound too high but not too low please just hear me out oh p***** i** magtanong ka nang ganun sa akin galit ako nasa droga ka pulis ka are laughter mabilog laughter tas magtanong siya kay diyan sa mga security sin of secretary drilon magpunta ka ng iloilo saan earn that much in so short a time how much is your salary kagaya si parojinog mayor of ozamiz you know if you are from ozamiz or you guys have been there ang ozamiz city is appears to be calm a there wala namang ano wala namang hunger o ano kaya pala lahat ng mga tagaozamiz natuto ng maging courier kuha sila ng supply doon ipagbili nila sa ibang lugar kaya karamihang nahuhuli puro taga this is really the meat of my the corruption so he was so rich that he could afford to kill two of his policemen no four pero hindi man nakita killing him tapos dalawang mig sa armed forces na intelligence until now hindi namin makita so ang power pumasok na talaga sa ulo niya and he has a house thing to see he had to go never never allow any soldier or police to go to prison for doing his duty i will not allow that sabi ko sige lang odicta the couple were killed in a sa pier were the ones do sila si mayor iyong mayor na malibog na hindi na umuwi dito sa right side it was espinosa the mayor who was killed while he was inside the detention cell when it was opened he tried to kill the policeman so there was a shootout ganun but whether true or not they will have my support hindi ko sila pababayaan talaga i will not abandon any sold ang pinaka delikado sa lahat sa atin the most dangerous the one dangerous thing is drugs i'm sure almost everybody of you listened to the presidential debates early on and you i know that you could maybe probably remember that i said a few words because of the very constrictive time given us to talk about our program bigyan ka lang ng one minute and seconds of what things can you kaya ako bullet nang bullet lang short lang ako wala na ako one was corruption second sabi ko drugs the third is corruption in the bureaucracy and fourth that i would try to look for money remember i never used the words build build build ireview ninyo 'yung tapes i never said it because i was not sure where would i go at that time i knew that china was really very angry at the behavior of the previous administration so hindi ko malaman so i never promised any grand or whatever so i just kept low and it was not because of that reason when i was there already 'yung intelligence ng or everybody the intelligence community warned me that there is something a brewing there in marawi they were passing arms openly and i thought that i needed more arms because the ones are really very old they are not accurate anymore a rifle is only good for rounds after that loose na 'yung bore malaki na paglabas ng bala wala ng direksyon sumasayaw na eh because there is that they have its own trajectory ang susunod umiikot 'yan because there is that rifling tawag nila so paglabas niyan umiikot 'yan sa may thousands every minute eh wala na handmedown 'yung sa pulis they get it from the army na talagang panahon pa ni marcos 'yung iba so i was a quandary i wanted to buy so i ordered from america ngayon bigla na lang nagsabi that ang america sabi ng mga congressman pati senador doon they took the floor to denounce me for my human rights violation daw so they refused to deliver what a cruel thing to do alam nila because i know that the americans are very very active sa intelligence and they know the situation ayaw nilang ibigay so i went to i sent word to china that i want to visit mr xi jinping so i went there i talked to the ambassador and i was received there i realized that we can talk things over and we can be in good stead again so sinabi kong problema ko sabi ni xi jinping it's ok i know your problem arms we'll give you the arms free napanganga ako sabi ko kasi i want to buy arms from you but i do not have any money right now could i just have a easy schedule for payment ah sabi niya you can have it so gusto ko nang marami kulang pa rin i went to russia sabi ni president putin oh we'll give it to you it was on that visit sa russia pumutok 'yung kaya talaga ako nag sentimiyento sa mga amerikano tapos nagpabili sila ng huey canada and look they told me that we will give you the helicopters but you cannot arm them you cannot place arm because you cannot use this against you own citizen sabi ko f*** you guys my own citizens are revolting trying to oust me out of office killing my soldiers and policemen and karamihan pinakamarami namamatay ngayon is barangay captains those who would opt to be sympathetic with the government 'yung ayaw nila pinapatay talaga nila sabi ko these communists they are my citizens right but they are walang ano i survived with a and to tell you frankly the rifle the sniper rifle sniper is used when your target is way way ahead very far the rifle that killed hapilon was a chinese made rifle sniper rifle and it was given to me in a glass case ng mga sundalo 'yun ang istorya diyan now itong ang corruption meron rin pero not so much dito sa military and even the police ang malaki dito 'yung mga mayaman the filthy rich sila 'yung mga may rebates sa kanilang taxes and if you will review them like the lopez's marcos was overthrown cory gave him back abscbn to rehabilitate abscbn oh anong ginawa niyo where did they get the money they borrowed from dbp ang dbp noon sikat eh noong naghiram tapos hindi nabayaran believe me these are government records they did not pay the dbp kaya ang dbp bumagsak and worst of all they condoned the utang 'yan ang masakit kaya ako kritikal diyan sa abscbn kaya ko pinup*******i** sila anong mukha ninyong magharap sa amin ng ganun now the franchise is up for renewal you ask me if i object yes i will object unanguna 'yung utang nila anuhin tablahin natin 'yan so how much money hanggang ngayon hindi natin alam wala 'yung record hindi namin mahanap so saan ako during the campaign because iwala naman akong pera may mga taong nagbigay sa akin when already i was up in the ratings nagdikit ako sa so money was already available and on the last week i had to pay that million magbayad ako may pera na eh 'yung ano ko na lang advertisement ko during the presidential debates every intermission makita mo 'yang on the side may mga may mga advertisement ako wala wala man akong pera pero nagkapera ako sabi ko meron akong ano alam mong ginawa ng kababuyan nitong abscbn inuna pa nila 'yung black propaganda ni trillanes na may mga bata ginamit which is prohibited by law you cannot use children there was this injunction and yet abscbn proceeded to air it ang akin wala marami man kami chiz escudero ako and maraming inotso ng abscbn ito kasing mga mayaman if you are small you commit crime you go to jail sa sigarilyo kaya sabi ko vista kaibigan ko 'yun and he offered to pay to file plunder charges against you and you will rot in prison i will see to it that so sabi how much i don't know about dominguez he's the one he's the one computing for the government you ask him ang dominguez nam thirtyseven billion ah bayad siya and of course you have this ayaw ko na lang kasi kaibigan na kami i mean we decided to let things be bygones be bygones but just to mention it ilang presidente dumaan ng pilipinas who was able to collect billion 'yan talagang i'll give you one example if you are connected with the company just let it pass itong line ng cellphone 'yung linyalinya man 'yang level parang air mga air ano tao i'm not a technical person pero it is given by the government free if you intend to build puro corruption pa rin 'yan there's no such thing as given bigyan ka niyan in the law it's free tapos magtayo ka anong tawag doon ah bandwidth binili ito w ibibigay lang talaga 'yan libre eh but somebody bought it here comes now the third i've been insisting on the third so that hanggang ngayon wala pang nangyari i have secretary rio who would fast track it tapos 'yun na favoringfavoring ganun sabi ko in a cabinet meeting kami sabi ko sige na kasi nagtatanong na in the hands of other persons the one that's with him has it been used no hinahawakan lang tapos the other one binili nila oh 'di sabi ko ibigay mo na 'yan doon sa ano sabi nila mayor ayaw isauli because they acquired it ah p***** i** tapos magbayad daw ako talalang tawagan mo lang npa laughter ten percent per tower tumba lahat p***** i** one billion igit mo l**** ka mga mayaman hindi naman itong soldiers to bring along things that belonging to the countries that they ou got it at the expense of the lives of so many innocent people and children you do not return the bell i will sabi niya na in the far and wide in between the years for the demand for their return kung governmentgovernment lang pero ang sabi ko cia ang cia ang laughter pa itong drugs ho i am now in a quandary you know drugs you might find me sometimes repugnant or repulsive but alam mo hindi ako sabi ko hindi ako nagyayabang ayaw mong mamatay huwag kang pumasok dito doon ka na lang sa o diyan ka be a listener there mahirap itong drugs kaya sabi ko i could not have made it more clearer if you destroy my country i will kill you if you make my children eat drugs use drugs and not make it more clearer to everybody to the human rights listen very carefully destroy my country i will kill you destroy our children i will kill you period no apologies no explanation no nothing i have said laid down the rules alam mo hindi lang ninyo alam sabi nila is the predicate of my discussion how narinig ninyo sa past administration even lining up none and when i not only them lahat naman but what is really obnoxious is they belong to the law enforcement ngayon pagsabi ko ito because nakita ko niyang surrender in hours hundreds then by the thousands at the end of the day the count of bato was ang count ni santiago was million at that time both of them are correct iloilo madam mga seven now i have million filipinos who are already slaves who cannot work anymore but lie there in a corner and when the monkey rides on the back they go out sell everything including their children the flesh of their unggoy eh talagang nag wala silang makita they kill especially the young people ayaw lang nang maniwala nitong mga human rights that they are really capable of killing how much ilang police ang nawala sa akin almost na pinatay ng mga durugista and you think of the itong sige magsabi thousand well in a way not then why would i worry with the son of a b**** there lying dead with a pool of blood when i think of the so many filipinos who are already ito this is the head of the family if he is hit then the wife will be in intended for food on the table is gone money little money intended for mga because ito wala na if he becomes dysfunctional the family if you have a friend or enemy gusto mong sirain bigyan mo ang anak ng shabu at makita ninyo and how many are those kasi ang shabu mahirap they are selling forever at you make it easy mga human right as if it is just an ordinary epidemic worse pagkatapos niyan lango na ang mgathey rape a family kill the father rape the mother the daughter kill all sa ombudsman kaya sabagay sa lugar ko nomagrape ka ng bata patayin mo judge huwag mo akong tanungin mayor judge ** ka eh wala na sira na one year use of shabu shrinks the brain if your brain has pahirapan mo ang tao so people students workers at night could no longer use these public streets publpaguwi passage among the dito sa mga squatters mga mo shabu is the commodity of the poor the rich idiots are on cocaine well ang mayaman mas mabuti because they come from a plant called poppy so it is not as toxic as itong shabu na chemical na para sa patay paghaluin nila sa ecstasy family man but you must consider that this is an issue of public interest i have to let it out in the open she has been using unfortunately the week before she was using ecstacy and she just dropped parang comatose pero umulit pa rin eh peer pressure talaga iyong colombia is still suffering no matter how many thousands of dollars have been poured in to correct an aberration in society until now britain america nagcontribute na you will make it hard now you do it in the ilagay mo na lang ang contamination diyan mga wala na talaga sabi ko the next president must be grappling with this issue ako if i cannot stop it during my term then i would even consider myself not a good public employee kaya sabi ko sa iba sabi ko sa military pati police t*** i** tapusin na natin ito during my term if i have to kill all of them i will kill all of them you know the presidency is a gift from god considering my circumstance when i won it you would hardly believe that i really won it fair and square so my gift to you and to your children is i will try my very best to wipe them from the face of the earth even applause even if i have to go to prison there is no perfect world here so you do you pay as a matter of fact i attempted twice so what do i ask in return nothing because i know that joining public service many many years ago i learned from my father who was governor of davao isa lang ang davao noon eh davao now comprises several local government units davao city that is how big actually my father said you know when you go public if you work for the government you are a worker of government and with it goes the kung anong mangyari sa iyo kaya kung alangan ka takot kang mamatay takot kang maambush ayaw pumasok diyan find then another vocation but i chose because i think work all these years before i became mayor in so hiningi ko eh return do not expect it when you are no longer there people would not rizal but then i have to call back the spaniards to do their thing here again laughter iyan ang so that is mdestroy our society or will make it difficult for us to live normally and so we and plaza or where they want to go return to the filipino people the right to go about anywhere ret na pumunta sila ng mga scenic without their heads being decapitated problems of the mi which is territorial in nature maybe when nur comes back we allowed him to go out to attend the oic the oil so i have only to face sa mindanao kung maplantsa ko si nur maplantsa ko itong npa if they can go down to my level mataas sila eh i admit that they are bright they are everything so mataas sila usap kami you give me the highfalutin words of marxism garbled hindi ko naintindihan hanggang lang po ako laughter kami ni dominguez ito si dominguez is the extreme always valedictorian ako ang nanay ko galit na galit sab laughter akyat naman ako roy cimatu is brilliant pma general i think he was good in pma up kaya hindi man man lang ako laughter may magna cum laude ba na magsagot ng but i said the strange the strange fic fact there is alam mo itong mga brilliant mga pmaer nandito presidential guard nandito silang p*** laughter and applause so huwag kayong magkoyears to finish high school magtanong kayo ng tagadavao iyan sila sonny matapos nito college ganun but when my father died doon ako nag doon ako nagtino talaga and there was a year na iyong insecurity na wala ka ng tatay so banat na talaga biro mo for the first time in my life naka ako laughter and applause totoo binigyan ako ni bilib siya sa akin t*** i** negotiable ko p***** i** sabi ko mahusay pala ito magaral tayo ah but you know you dyour time studying wala it breaks the spirit of youlaughter laro laro muna laughter so to the presidency i hope when you are here use your talent lahat na nang natutunan kong kalokohan ginamit ko na kaya nagwarning ko talaga huwag na huwag talaga kayong t*** i** patay kayo sa akin i will skin you alive not for me ano man makuha beh hoy human rights listen to me anong makuha ko what would i get killing innocent people what may isa kaming propesor justice solidum mimics voice of justice solidum people g**d*** it when i retire i have to look for a place because they laughter ganun si justice solidum eh ito nagdaan ito sila may propesor kami si justice solidum ilonggo ito oh si judge clavecilla presidential adviser na ngayon huwag mo gyud bitawan laughter womanizing laughter meron siya sa diyan sa holy spirit laughter centro escolar he has a suffering from a syndrome until now hindi talaga totoo so i would like to just say to you that kasali na ang kabastusan diyan but that is ect because i never transitioned from being called mayor not really the one not because i want to ignore you nasanay ako sa mayor mayor lang ako hanggang ngayon ang ugali ko mayor ang style ko mayor ang bunganga ko bastos kasi doon kung hindi ka p******** i** mo wala namang maniwala sa iyo ganun talaga ngayon minsan sa crowd but hindi raw ako statesman sabi ng mga elite dito classmate man kami nito oh ikaw meron ba tayong subject na statesmanship laughter kaya wala tayong natutuhan diyan gusto kong sabihin f*** you iyan ang galit eh but behind that really meron akong hia sense of love for my country is very strong maski tagamindanao lang kami ha my mother is halfmaranao papa ko from cebu danao but my grandfather was chinese mayari ng ba so many bakeries in marawi and iligan noon may konti ho akong but i assure you i have the minimum of values to run a country at least better than the others because wala akong kalokohan applause alam mo nandiyan man sila lahat they are all here no transaction of reclamation na kung ano approval of contracts it does not reach my table i have prohibited that kayo kaya nga abscbn sabihin ko wala kayong makalkal sa akin sa pagkatao ko basta pera because maski piso the contracts the documents begins and ends there sa malaca sa cabinet member wala talaga ako so what i get is my salary but you know that sabihin ninyo because i have two families laughter i have to support the grandmother i i challenge you your one peso will win one million sa akin wala kang makita sa malacaan nagpirma ako ng pera either representation o ano even clothing allowance wala ako mapreso man ako it could not be corruption that would be very impossible mine maraming salamat po applause end\""
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[200]['clean_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}