2_web_html_to_csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4 # TODO: Achar o motto do bs4\n",
    "from bs4.element import NavigableString\n",
    "import re, logging\n",
    "import dateutil.parser\n",
    "import pandas as pd\n",
    "from collections import namedtuple\n",
    "import settings\n",
    "\n",
    "logging.basicConfig(level=logging.WARNING)\n",
    "log = logging.getLogger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Msg = namedtuple('Msg', 'text name datetime quote')\n",
    "SubjectChange = namedtuple('SubjectChange', 'subject phone datetime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{settings.GROUP_NAME}.html') as f:\n",
    "#with open(f'sample.html') as f:\n",
    "    html = f.read()\n",
    "    soup = bs4.BeautifulSoup(html, 'html5lib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_messages(soup): # TODO: Emojis, Images, Quotes, Audios\n",
    "    msgs, subject_changes = [], []\n",
    "    date, phone = None, None\n",
    "    global DEBUG\n",
    "    body = soup.find('body')\n",
    "    for div in body.select('[data-pre-plain-text]'):\n",
    "        log.debug('------------------')\n",
    "        log.debug(div)\n",
    "        datetime_author = div.attrs['data-pre-plain-text']\n",
    "        match = re.search(r'\\[(.*)\\] (.*): ', div.attrs['data-pre-plain-text'])\n",
    "        datetime, author = match.groups()\n",
    "        datetime = dateutil.parser.parse(datetime)\n",
    "        quote = div.select('span.quoted-mention')\n",
    "        if quote:\n",
    "            quote = quote[0].parent.parent\n",
    "            #DEBUG = quote\n",
    "            quoted_person = quote.find('span').text\n",
    "            quote.decompose()\n",
    "        else:\n",
    "            quoted_person = None\n",
    "        DEBUG = div\n",
    "        text = div.select('span.selectable-text')\n",
    "        if text:\n",
    "            for i in text[0].select('img.emoji'):\n",
    "                i.replace_with(i.attrs['alt'])\n",
    "            text = text[0].text\n",
    "        else:\n",
    "            text = div.select('div.selectable-text img')[0].attrs['alt']\n",
    "        msg = Msg(text, author, datetime, quoted_person)\n",
    "        log.info(f'Got a msg {msg}')\n",
    "        msgs.append(msg)  \n",
    "    return msgs\n",
    "\n",
    "\n",
    "messages = get_messages(soup)\n",
    "messages[:20] # TODO: Resolver quotes q estao vazando no texto dos outros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%script echo skip\n",
    "a = DEBUG.select('span.selectable-text')\n",
    "b = a[0].select('img.emoji')[0]\n",
    "t = b.attrs['alt']\n",
    "b.replace_with(t)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(messages)\n",
    "df.to_csv(f'{settings.GROUP_NAME}_messages.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.quote.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.text.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}