02_clean_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "# nltk.download('stopwords')\n",
    "# nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from scraper_functions import has_no_words\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords = stopwords.words('english')\n",
    "# stopwords.remove('no')\n",
    "# stopwords.remove('not')\n",
    "fil_stopwords = ['ang', 'po', 'naman', 'ito', 'medyo', 'may', 'yung', 'kasi', 'sa', 'na', 'ng', 'ko', 'ba', 'ako', 'lang']\n",
    "for word in fil_stopwords:\n",
    "    stopwords.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(455, 3)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_name = 'tika_transcript'\n",
    "\n",
    "df = pd.read_csv('data/{}.csv'.format(file_name), index_col=0)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-10-05</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-09-28</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Statement of President Rodrigo Roa Duterte dur...</td>\n",
       "      <td>2020-09-22</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-09-21</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-09-14</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-09-07</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Statement of President Rodrigo Roa Duterte at ...</td>\n",
       "      <td>2020-09-02</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Speech of President Rodrigo Roa Duterte during...</td>\n",
       "      <td>2020-08-31</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Talk to the People of President Rodrigo Roa Du...</td>\n",
       "      <td>2020-08-31</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Talk to the People of   President Rodrigo Roa ...</td>\n",
       "      <td>2020-08-25</td>\n",
       "      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title       date  \\\n",
       "0  Talk to the People of President Rodrigo Roa Du... 2020-10-05   \n",
       "1  Talk to the People of President Rodrigo Roa Du... 2020-09-28   \n",
       "2  Statement of President Rodrigo Roa Duterte dur... 2020-09-22   \n",
       "3  Talk to the People of President Rodrigo Roa Du... 2020-09-21   \n",
       "4  Talk to the People of President Rodrigo Roa Du... 2020-09-14   \n",
       "5  Talk to the People of President Rodrigo Roa Du... 2020-09-07   \n",
       "6  Statement of President Rodrigo Roa Duterte at ... 2020-09-02   \n",
       "7  Speech of President Rodrigo Roa Duterte during... 2020-08-31   \n",
       "8  Talk to the People of President Rodrigo Roa Du... 2020-08-31   \n",
       "9  Talk to the People of   President Rodrigo Roa ... 2020-08-25   \n",
       "\n",
       "                                                text  \n",
       "0  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "1  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "2  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "3  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "4  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "5  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "6  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "7  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "8  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  \n",
       "9  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['date'] = pd.to_datetime(df['date'], format='%B %d, %Y')\n",
    "df = df.sort_values('date', ascending=False).reset_index().drop('index', axis=1)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speech of President Duterte during the mass oath taking of the newly elected officers of the Federation of Filipino-Chinese Chambers of Commerce and Industry, Inc. (FFCCCII)\n",
      " \n",
      "  \n",
      "  \n",
      "Ladies and gentlemen, I am very proud to administer your oath of office. \n",
      "You choose the venue and good evening ma’am, President Ramos and... \n",
      "  \n",
      "[oath-taking ceremony] \n",
      "  \n",
      "Maupo muna kayo sandali. I was told just to administer your oath and that’s \n",
      "it. Eh sabi ko, just --- tatlo ito, the mayors and governors and ito without \n",
      "speeches. \n",
      "  \n",
      "Sabi ko, look I did study foreign service, totoo ‘yan. Hindi dini-discredit. I \n",
      "took up diplomatic practice and procedure. So it go\n",
      "===============================\n",
      "d sabi ko sa \n",
      "iyo, I will protect you. I will protect not only Chinese. Everybody. Whether \n",
      "it's German or doing business dito. \n",
      "  \n",
      "Ayaw ko talaga ng kalokohan. Ayaw ko 'yang mag sige... And 'yung \n",
      "mayroon kayo diyan na historical na 'yang hingi niya, tell me para 'yung \n",
      "mga anak ninyo balang araw aabutin pa 'yan eh, hanggang 30 years 'yan \n",
      "siguro --- 65 years old. At least makonti --- ma-menos-menos ang \n",
      "corruption ng Pilipinas. \n",
      "  \n",
      "At kayo naman, kumportable na... Maski na kasi --- maski 10 pesos basta \n",
      "'yung pinapadaan sa ganun, masakit 'yan eh kasi pinagpaguran. \n",
      "  \n",
      "I know that for a worthy cause sa kapwa tao ninyo pwede kang magtapon \n",
      "ng isang milyon. Pero ‘pag hiningian ka ng singkwenta pesos tapos 'yung \n",
      "dala takot, nakaka-init ng ulo. \n",
      "  \n",
      "Now is your time. Now is the time for all lalo na kayong mga Chinese \n",
      "historical victims of corruption sa gobyerno. Ngayon na ninyo isabi 'yan para \n",
      "ma-menos-menosan natin lahat. \n",
      "  \n",
      "Maraming salamat po. [applause] \n",
      "\n",
      "\n",
      "\n",
      "  \n",
      "  \n",
      "\n",
      "--- END --- \n",
      "\t\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Example of speech; remove everything before the date\n",
    "entry = df.loc[150]\n",
    "sample = entry['text']\n",
    "index = sample.find(']')\n",
    "print(entry['title'])\n",
    "print(sample[index+1:1000])\n",
    "print('===============================')\n",
    "print(sample[-1000:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(text):\n",
    "    titles = ['mr', 'dr', 'mrs', 'atty', 'ms', 'inc']\n",
    "    titles_str = ''.join([f'(?<!({x}))' for x in titles])\n",
    "    \n",
    "    if not isinstance(text, float):\n",
    "        # Remove headings like \"PRESIDENT DUTERTE:\"\n",
    "        text = re.sub(r'[A-Z ]+: ?', '', text)\n",
    "        \n",
    "        text = text.lower()\n",
    "        \n",
    "        # Remove words inside brackets, e.g. [laughter], [applause]\n",
    "        text = re.sub(r'\\[.*?\\]', '', text)\n",
    "        \n",
    "#         Remove punctuation\n",
    "        text = re.sub(r'([.!?])', ' ', text)\n",
    "\n",
    "#         I included the asterisk * to include swears\n",
    "        text = re.sub(r'([^A-Z a-z*\\'\\-])', '', text)\n",
    "#         text = re.sub(r'([^A-Z a-z*\\'\\-.!?])', '', text)\n",
    "    \n",
    "#         Remove hyphens EXCEPT if they're preceded by a letter\n",
    "#         or if they are group as three hyphens\n",
    "        text = re.sub(r'(?<!-)(-{1,2}|-{4,})(?!-)', '', text)\n",
    "\n",
    "#         Not sure if I will still remove stopwords\n",
    "#         text = ' '.join([word for word in text.split() if word not in stopwords])\n",
    "\n",
    "        text = text.replace('presidential news desk', '')\n",
    "        text = text.replace('pcoopnd', '')\n",
    "        \n",
    "        text = ' '.join(text.split())\n",
    "        \n",
    "#         Remove puncutation\n",
    "#         text = re.sub(r'\\.{3}', ' ', text)\n",
    "#         text = text[:-1]\n",
    "\n",
    "#         period_reg = titles_str + '\\. '\n",
    "#         text = re.sub(period_reg, ' . ', text)\n",
    "#         text = re.sub('\\? ', ' ? ', text)\n",
    "#         text = re.sub('! ', ' ! ', text)\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_swears(text):\n",
    "    if not isinstance(text, float):\n",
    "        text = text.replace('kat*********', 'katarantaduhang')\n",
    "        text = text.replace('k*********', 'katarantaduhan')\n",
    "        text = text.replace('ka****han', 'kagaguhan')\n",
    "        text = text.replace('ka*******', 'kagaguhan')\n",
    "        text = text.replace('k********', 'kagaguhan')\n",
    "        text = text.replace('t**********', 'tarantadong')\n",
    "        text = text.replace('t********', 'tarantado')\n",
    "        text = text.replace('t****** i****', 'tanginang')\n",
    "        text = text.replace('t**** i****', 'tanginang')\n",
    "        text = text.replace('t*****i***', 'tanginang')\n",
    "        text = text.replace('t****i***', 'tanginang')\n",
    "        text = text.replace('t*** i*****', 'tanginang')\n",
    "        text = text.replace('t*** i****', 'tangina')\n",
    "        text = text.replace('t*** i***', 'tangina')\n",
    "        text = text.replace('t****i**', 'tangina')\n",
    "        text = text.replace('t****i**', 'tangina')\n",
    "        text = text.replace('t****i***', 'tanginang')\n",
    "        text = text.replace('t**** i****', 'tangina')\n",
    "        text = text.replace('t***** i**', 'tangina')\n",
    "        text = text.replace('t**** i***', 'tangina')\n",
    "        text = text.replace('t**** i**', 'tangina')\n",
    "        text = text.replace('t*** i**', 'tangina')\n",
    "        text = text.replace('t** i**', 'tangina')\n",
    "        text = text.replace('t*** i*', 'tangina')\n",
    "        text = text.replace('t*******', 'tarantado')\n",
    "        text = text.replace('t******', 'tarantado')\n",
    "        text = text.replace('t*****', 'taragis')\n",
    "        text = text.replace('t****', 'tubol')\n",
    "        text = text.replace('t***', 'tangina')\n",
    "        text = text.replace('t**', 'tae')\n",
    "        text = text.replace('m***********', 'motherfucker')\n",
    "        text = text.replace('m***** f*****', 'motherfucker')\n",
    "        text = text.replace('magp*********', 'mag-puputa-puta')\n",
    "        text = text.replace('p********** i**', 'putangina')\n",
    "        text = text.replace('p********* i**', 'putangina')\n",
    "        text = text.replace('p******** i**', 'putangina')\n",
    "        text = text.replace('p********', 'putangina')\n",
    "        text = re.sub(r't{1}\\*+ ?i\\*{3,}', 'tanginang', text)\n",
    "        text = re.sub(r'p{1}\\*+ ?i\\*{3,}', 'putanginang', text)\n",
    "        text = re.sub(r't{1}\\*+ ?i\\*{1,2}', 'tangina', text)\n",
    "        text = re.sub(r'p{1}\\*+ ?i\\*{1,2}', 'putangina', text)\n",
    "        text = text.replace('k*** i**', 'kingina')\n",
    "        text = text.replace('l*****i****', 'lecheleche')\n",
    "        text = re.sub(r'l\\*{6,}', 'lecheng', text)\n",
    "        text = re.sub(r'l\\*{2,5}', 'leche', text)\n",
    "        text = text.replace('mam***** i**', 'mamutangina')\n",
    "        text = text.replace('mam**** i**', 'mamutangina')\n",
    "        text = text.replace('mamutangi**', 'mamutangina')\n",
    "        text = text.replace('mum****** i**', 'mamutangina')\n",
    "        text = text.replace('q** s* j***', 'que se joda')\n",
    "        text = text.replace('y******', 'yaring')\n",
    "        text = text.replace('y*****', 'yaring')\n",
    "        text = text.replace('y****', 'yari')\n",
    "        text = text.replace('y***', 'yari')\n",
    "        text = text.replace('kag*******', 'kagaguhang')\n",
    "        text = text.replace('kap*******', 'kaputahan')\n",
    "        text = text.replace('kap******', 'kaputahan')\n",
    "        text = text.replace('pinu**** i**', 'pinutangina')\n",
    "        text = text.replace('p*******', 'putangina')\n",
    "        text = text.replace('p******', 'puchang')\n",
    "        text = text.replace('p*****', 'putang')\n",
    "        text = text.replace('p****', 'puta')\n",
    "        text = text.replace('p***', 'puta')\n",
    "        text = text.replace('p**', 'puta')\n",
    "        text = text.replace('p*', 'puta')\n",
    "        text = text.replace('*****i**', 'idiots')\n",
    "        text = text.replace('i*****', 'idiots')\n",
    "        text = text.replace('i****', 'idiot')\n",
    "        text = text.replace('i***', 'igit')\n",
    "        text = text.replace('god d******', 'goddamn')\n",
    "        text = text.replace('g**d****', 'goddamn')\n",
    "        text = text.replace('g** d***', 'goddamn')\n",
    "        text = text.replace('g**d***', 'goddamn')\n",
    "        text = text.replace('godd*****', 'goddamn')\n",
    "        text = text.replace('godd****', 'goddamn')\n",
    "        text = text.replace('god****', 'goddamn')\n",
    "        text = text.replace('g**d****', 'goddamn')\n",
    "        text = text.replace('g********', 'goddamn')\n",
    "        text = text.replace('g*******', 'goddamn')\n",
    "        text = text.replace('g******', 'goddamn')\n",
    "        text = text.replace('g*****', 'gagong')\n",
    "        text = text.replace('g****', 'gago')\n",
    "        text = text.replace('o***', 'tite')\n",
    "        text = text.replace('g***', 'gago')\n",
    "        text = text.replace('g**o', 'gago')\n",
    "        text = text.replace('g**a', 'gaga')\n",
    "        text = text.replace('g*', 'gago')\n",
    "        text = text.replace('b**** s**** i**', 'bullshitting')\n",
    "        text = text.replace('b**** s*** i**', 'bullshitting')\n",
    "        text = text.replace('b**********', 'bullshitting')\n",
    "        text = text.replace('b*** s*****', 'bullshitted')\n",
    "        text = text.replace('b*b*** s******', 'bullshit')\n",
    "        text = text.replace('b**s***ted', 'bullshitted')\n",
    "        text = text.replace('b*********', 'binullshit')\n",
    "        text = text.replace('bin****s***', 'binullshit')\n",
    "        text = text.replace('bin*** s***', 'binullshit')\n",
    "        text = text.replace('b*** s****', 'bullshit')\n",
    "        text = text.replace('b*** s***', 'bullshit')\n",
    "        text = text.replace('b*** s**', 'bullshit')\n",
    "        text = text.replace('b*b******', 'bullshit')\n",
    "        text = text.replace('b*******', 'bullshit')\n",
    "        text = text.replace('b******', 'bitches')\n",
    "        text = text.replace('b*****', 'bobong')\n",
    "        text = text.replace('s** of a b****', 'bitch')\n",
    "        text = text.replace('b****', 'bitch')\n",
    "        text = text.replace('b***', 'bobo')\n",
    "        text = text.replace('c****c****', 'chukchak')\n",
    "        text = text.replace('c***c***', 'chukchak')\n",
    "        text = text.replace('s********', 'shitting')\n",
    "        text = text.replace('s******', 'salsalin')\n",
    "        text = text.replace('s*****', 'shitty')\n",
    "        text = text.replace('s****', 'shit')\n",
    "        text = text.replace('s***', 'shit')\n",
    "        text = text.replace('sh**', 'shit')\n",
    "        text = text.replace('s**', 'shit')\n",
    "        text = text.replace('a******', 'asshole')\n",
    "        text = text.replace('a*****', 'asshole')\n",
    "        text = text.replace('a**h***', 'asshole')\n",
    "        text = text.replace('a**', 'ass')\n",
    "        text = text.replace('d***', 'damn')\n",
    "        text = text.replace('k****', 'kiyud')\n",
    "        text = text.replace('u******', 'ukinam')\n",
    "        text = text.replace('u*****', 'ukinam')\n",
    "        text = text.replace('u****', 'ulol')\n",
    "        text = text.replace('u***', 'ulol')\n",
    "        text = text.replace('u**', 'ulol')\n",
    "        text = text.replace('w*****', 'whore')\n",
    "        text = text.replace('w****', 'whore')\n",
    "        text = text.replace('h******', 'hindot')\n",
    "        text = text.replace('v******', 'vagina')\n",
    "        text = text.replace('v*****', 'vagina')\n",
    "        text = text.replace('f*** y**', 'fuck you')\n",
    "        text = re.sub(r'f{1}\\*{5,}', 'fucking', text)\n",
    "        text = re.sub(r'f{1}\\*{2,4}', 'fuck', text)\n",
    "        text = text.replace('i**', 'ina')\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import tokenize\n",
    "\n",
    "# Split the text into a list of sentences\n",
    "def split_sentences(text):\n",
    "    if not isinstance(text, float):\n",
    "        index_of_date = text.find(']')\n",
    "        text = text[index_of_date+1:]\n",
    "        text = ' '.join(text.split())\n",
    "        return tokenize.sent_tokenize(text)\n",
    "    \n",
    "    return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = df['text'].apply(split_sentences).sum()\n",
    "df_clean = pd.DataFrame(sentences, columns=['text'])\n",
    "df_clean['clean_text'] = df_clean['text'].apply(clean_data)\n",
    "df_clean = df_clean.drop_duplicates()\n",
    "df_clean['clean_text'] = df_clean['clean_text'].apply(replace_swears)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INDEX: 453\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'i was a special prosecutor for the tanodbayan ang predecessor ng ombudsman'"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "index = random.randint(0, df.shape[0])\n",
    "print('INDEX: ' + str(index))\n",
    "' '.join(df_clean.loc[index]['clean_text'].split()[:300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I do not have to observe what are the kinds of sh** that you are trying to foist to me about non-arrest and everything.'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.loc[169557]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26715</th>\n",
       "      <td>Mabasa ng judge 'yun tapos ikaw ang fiscal, b*...</td>\n",
       "      <td>mabasa ng judge 'yun tapos ikaw ang fiscal b*b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28867</th>\n",
       "      <td>What’s the f*.</td>\n",
       "      <td>whats the f*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62498</th>\n",
       "      <td>Unya gal*-l*.</td>\n",
       "      <td>unya gal*l*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86884</th>\n",
       "      <td>Kay og mayor ka didto sa Danao --- ah sa Minda...</td>\n",
       "      <td>kay og mayor ka didto sa danao --- ah sa minda...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87428</th>\n",
       "      <td>Nagkal*****-l* --- bitaw.</td>\n",
       "      <td>nagkalechel* --- bitaw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91792</th>\n",
       "      <td>Ang p*********.</td>\n",
       "      <td>ang putangina*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92200</th>\n",
       "      <td>Kaya sabiko, nung b******** ko si Obama.</td>\n",
       "      <td>kaya sabiko nung bullshit* ko si obama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104066</th>\n",
       "      <td>These Human Rights are m************.</td>\n",
       "      <td>these human rights are motherfucker*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105743</th>\n",
       "      <td>Ay b******** i**.</td>\n",
       "      <td>ay bullshit* ina</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108339</th>\n",
       "      <td>Ay m***** f*****.</td>\n",
       "      <td>ay m***** fucking</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108498</th>\n",
       "      <td>That m***** f***** who said on TV --- I guess ...</td>\n",
       "      <td>that m***** fucking who said on tv --- i guess...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122810</th>\n",
       "      <td>S*******.</td>\n",
       "      <td>salsalin*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136881</th>\n",
       "      <td>L*- **** talaga nila ‘yang p***** i**** bata h...</td>\n",
       "      <td>l* **** talaga nila yang putanginang bata hang...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143477</th>\n",
       "      <td>Ginal*-l* gud nila na hangtod --- para hanggan...</td>\n",
       "      <td>ginal*l* gud nila na hangtod --- para hanggang...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144312</th>\n",
       "      <td>Sinabi ko, “P***** i**** mga ‘to, l**** kayo.”...</td>\n",
       "      <td>sinabi ko putanginang mga to leche kayo hindi ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150771</th>\n",
       "      <td>Mal*-l***** ang bayan ko and it would not be f...</td>\n",
       "      <td>mal*leche ang bayan ko and it would not be fai...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "26715   Mabasa ng judge 'yun tapos ikaw ang fiscal, b*...   \n",
       "28867                                      What’s the f*.   \n",
       "62498                                       Unya gal*-l*.   \n",
       "86884   Kay og mayor ka didto sa Danao --- ah sa Minda...   \n",
       "87428                           Nagkal*****-l* --- bitaw.   \n",
       "91792                                     Ang p*********.   \n",
       "92200            Kaya sabiko, nung b******** ko si Obama.   \n",
       "104066              These Human Rights are m************.   \n",
       "105743                                  Ay b******** i**.   \n",
       "108339                                  Ay m***** f*****.   \n",
       "108498  That m***** f***** who said on TV --- I guess ...   \n",
       "122810                                          S*******.   \n",
       "136881  L*- **** talaga nila ‘yang p***** i**** bata h...   \n",
       "143477  Ginal*-l* gud nila na hangtod --- para hanggan...   \n",
       "144312  Sinabi ko, “P***** i**** mga ‘to, l**** kayo.”...   \n",
       "150771  Mal*-l***** ang bayan ko and it would not be f...   \n",
       "\n",
       "                                               clean_text  \n",
       "26715   mabasa ng judge 'yun tapos ikaw ang fiscal b*b...  \n",
       "28867                                        whats the f*  \n",
       "62498                                         unya gal*l*  \n",
       "86884   kay og mayor ka didto sa danao --- ah sa minda...  \n",
       "87428                              nagkalechel* --- bitaw  \n",
       "91792                                      ang putangina*  \n",
       "92200              kaya sabiko nung bullshit* ko si obama  \n",
       "104066               these human rights are motherfucker*  \n",
       "105743                                   ay bullshit* ina  \n",
       "108339                                  ay m***** fucking  \n",
       "108498  that m***** fucking who said on tv --- i guess...  \n",
       "122810                                          salsalin*  \n",
       "136881  l* **** talaga nila yang putanginang bata hang...  \n",
       "143477  ginal*l* gud nila na hangtod --- para hanggang...  \n",
       "144312  sinabi ko putanginang mga to leche kayo hindi ...  \n",
       "150771  mal*leche ang bayan ko and it would not be fai...  "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean[df_clean['clean_text'].str.contains(r'[A-Za-z]{1}\\*+', regex=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove translations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving cleaned text as CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove NaN values\n",
    "df_clean = df_clean.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PRESIDENT RODRIGO ROA DUTERTE: Good day, my co...</td>\n",
       "      <td>good day my countrymen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>We are here gathered to render a report weekly...</td>\n",
       "      <td>we are here gathered to render a report weekly...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kaya kami nandito at marami namang haka-haka p...</td>\n",
       "      <td>kaya kami nandito at marami namang hakahaka pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gawain mo na, masama ka pa rin.</td>\n",
       "      <td>gawain mo na masama ka pa rin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kung hindi mo gawin, mas lalo kang masama.</td>\n",
       "      <td>kung hindi mo gawin mas lalo kang masama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  PRESIDENT RODRIGO ROA DUTERTE: Good day, my co...   \n",
       "1  We are here gathered to render a report weekly...   \n",
       "2  Kaya kami nandito at marami namang haka-haka p...   \n",
       "3                    Gawain mo na, masama ka pa rin.   \n",
       "4         Kung hindi mo gawin, mas lalo kang masama.   \n",
       "\n",
       "                                          clean_text  \n",
       "0                             good day my countrymen  \n",
       "1  we are here gathered to render a report weekly...  \n",
       "2  kaya kami nandito at marami namang hakahaka pa...  \n",
       "3                      gawain mo na masama ka pa rin  \n",
       "4           kung hindi mo gawin mas lalo kang masama  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.to_csv('data/{}_sentences.csv'.format(file_name))\n",
    "df_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(174346, 2)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dutertespeech",
   "language": "python",
   "name": "dutertespeech"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}