pre_processing_20_news.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/rgomes/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "Automatically created module for IPython interactive environment\n"
     ]
    }
   ],
   "source": [
    "#Importing the dependencies\n",
    "import re, string, unicodedata, itertools, random, os, glob, math, time, json\n",
    "import pickle\n",
    "import nltk\n",
    "from nltk import SnowballStemmer\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import silhouette_samples, silhouette_score\n",
    "from scipy.spatial import distance_matrix\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer, CountVectorizer\n",
    "\n",
    "from __future__ import print_function\n",
    "print(__doc__)\n",
    "\n",
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "plt.rcParams['figure.figsize'] = (24, 9)\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pré processing the corpora\n",
    "The first step that we used was to remove special characters\n",
    "such as ”#”, ”@”, and ”/” and irrelevant information such\n",
    "as email addresses and numbers.\n",
    "\n",
    "Then, we used\n",
    "tokenization the text to read words and check whether these\n",
    "were relevant or irrelevant term. We ignored the subject header,\n",
    "email addresses, numbers, and punctuation\n",
    "The next step that we applied was the removal of stop\n",
    "words.\n",
    "\n",
    "The third step was a word stemmer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_non_ascii(words):\n",
    "    \"\"\"Remove non-ASCII characters from list of tokenized words\"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        new_word = unicodedata.normalize('NFKD', word).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n",
    "        new_words.append(new_word)\n",
    "    return new_words\n",
    "\n",
    "def to_lowercase(words):\n",
    "    \"\"\"Convert all characters to lowercase from list of tokenized words\"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        new_word = word.lower()\n",
    "        new_words.append(new_word)\n",
    "    return new_words\n",
    "\n",
    "def remove_punctuation(words):\n",
    "    \"\"\"Remove punctuation from list of tokenized words\"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        new_word = re.sub(r'[^\\w\\s]', '', word)\n",
    "        if new_word != '':\n",
    "            new_words.append(new_word)\n",
    "    return new_words\n",
    "\n",
    "def replace_numbers(words):\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        new_word = re.sub(r'[0-9]', '', word)\n",
    "        if new_word != '':\n",
    "            new_words.append(new_word)\n",
    "    return new_words\n",
    "\n",
    "def remove_stopwords(words):\n",
    "    \"\"\"Remove stop words from list of tokenized words\"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        if word not in stopwords.words('english'):\n",
    "            new_words.append(word)\n",
    "    return new_words\n",
    "\n",
    "def stemming(words):\n",
    "    \"\"\"Reduce words to radical\"\"\"\n",
    "    stemmer = SnowballStemmer('english')\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        new_word = stemmer.stem(word)\n",
    "        new_words.append(new_word)\n",
    "    return new_words\n",
    "\n",
    "def trimAndClear(words):    \n",
    "    \"\"\"Remove words with only 2 letters, whitespace and empty\"\"\"\n",
    "    new_words = map(lambda w : w.strip(), words)\n",
    "    new_words = list(filter(lambda w: w != '' and len(w) > 2, new_words))\n",
    "    return new_words\n",
    "\n",
    "def normalize(words):\n",
    "    words = remove_non_ascii(words)\n",
    "    words = to_lowercase(words)\n",
    "    words = remove_punctuation(words)\n",
    "    words = replace_numbers(words)\n",
    "    words = remove_stopwords(words)\n",
    "    words = stemming(words)\n",
    "    words = trimAndClear(words)\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadCorpus(files):\n",
    "    voc = set()\n",
    "    docs = []\n",
    "    # build the vocabulary\n",
    "    for filename in files:\n",
    "       \n",
    "        with open(filename, 'rb') as f:\n",
    "            print('Loading', filename)\n",
    "            index = 0\n",
    "            tmp = []\n",
    "            for line in f:\n",
    "                # readline\n",
    "                line = line.decode(\"utf-8\", \"ignore\")\n",
    "\n",
    "                if re.match('^(Newsgroup:)', line):\n",
    "                    # email found\n",
    "\n",
    "                    # save the full email readble\n",
    "                    email = ' '.join(list(itertools.chain.from_iterable(tmp)))\n",
    "                    if email != ' ':\n",
    "                        docs.append(email)\n",
    "\n",
    "                    index = 0\n",
    "                    tmp = []\n",
    "\n",
    "                index += 1\n",
    "\n",
    "                # achei o header\n",
    "                # pula 3 linhas\n",
    "                if index > 4:\n",
    "                    # read the email\n",
    "                    tokens = line.split(' ')\n",
    "                    words = normalize(tokens)\n",
    "\n",
    "                    voc.update(words)\n",
    "                    tmp.append(words)       \n",
    "\n",
    "    # save as pickle\n",
    "    print('Saving', len(voc), 'unique words')\n",
    "    tokens_file = open('./representations/tokens_vector.pkl', 'wb')\n",
    "    pickle.dump(list(voc), tokens_file)\n",
    "    print('saved...')\n",
    "    \n",
    "    print('Saving', len(docs), 'documents')\n",
    "    d_file = open('./representations/documents.pkl', 'wb')\n",
    "    pickle.dump(list(docs), d_file)\n",
    "    print('saved...')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tf(term, document):\n",
    "    return freq(term, document)\n",
    "\n",
    "def freq(term, document):\n",
    "    return document.split().count(term)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome to Preprocessing\n",
      "Loading 20_news/sci.space.txt\n",
      "Loading 20_news/talk.religion.misc.txt\n",
      "Loading 20_news/talk.politics.misc.txt\n",
      "Loading 20_news/rec.sport.baseball.txt\n",
      "Saving 34666 unique words\n",
      "saved...\n",
      "Saving 6768 documents\n",
      "saved...\n",
      "Time to preprocess all files: 312.31193804740906\n"
     ]
    }
   ],
   "source": [
    "print('Welcome to Preprocessing')\n",
    "files = [\n",
    "    '20_news/sci.space.txt',\n",
    "    '20_news/talk.religion.misc.txt',\n",
    "    '20_news/talk.politics.misc.txt',\n",
    "    '20_news/rec.sport.baseball.txt'\n",
    "]\n",
    "    \n",
    "t0 = time.time()\n",
    "loadCorpus(files)\n",
    "print('Time to preprocess all files:', (time.time()-t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34666 tokens loaded\n",
      "6768 tokens loaded\n"
     ]
    }
   ],
   "source": [
    "data_f = open('./representations/tokens_vector.pkl', 'rb')\n",
    "voc = pickle.load(data_f)\n",
    "print(len(voc), 'tokens loaded')\n",
    "\n",
    "data_f = open('./representations/documents.pkl', 'rb')\n",
    "docs = pickle.load(data_f)\n",
    "print(len(docs), 'tokens loaded')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "calculating tf matrix...\n"
     ]
    }
   ],
   "source": [
    "tf_matrix = []\n",
    "\n",
    "print('calculating tf matrix...')\n",
    "for doc in docs:\n",
    "    \n",
    "    #print('The doc is \"' + doc + '\"')\n",
    "    tf_vector = [tf(word, doc) for word in voc]\n",
    "    #tf_vector_string = ', '.join(format(freq, 'd') for freq in tf_vector)\n",
    "    #print('The tf vector for Document %d is [%s]' % ((docs.index(doc)+1), tf_vector_string))\n",
    "    tf_matrix.append(tf_vector)\n",
    "    \n",
    "print(np.shape(tf_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('saving tf matrix...')\n",
    "np.savetxt('./representations/tf_matrix.txt', tf_matrix)\n",
    "print('saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def l2_normalizer(vec):\n",
    "    denom = np.sum([el**2 for el in vec])\n",
    "    if (denom != 0):\n",
    "        return [(el / math.sqrt(denom)) for el in vec]\n",
    "    else: return [(el for el in vec)]\n",
    "\n",
    "doc_term_matrix_l2 = []\n",
    "for vec in tf_matrix:\n",
    "    doc_term_matrix_l2.append(l2_normalizer(vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<generator object l2_normalizer.<locals>.<genexpr> at 0x7f767faef780>]\n",
      "saving tf matrix normalized...\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Mismatch between array dtype ('object') and format specifier ('%.18e')",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/numpy/lib/npyio.py\u001b[0m in \u001b[0;36msavetxt\u001b[0;34m(fname, X, fmt, delimiter, newline, header, footer, comments, encoding)\u001b[0m\n\u001b[1;32m   1370\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1371\u001b[0;31m                     \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mformat\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnewline\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1372\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: must be real number, not list",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-123-35aeebaeaffd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc_term_matrix_l2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'saving tf matrix normalized...'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavetxt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./representations/tf_matrix_normalized.txt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdoc_term_matrix_l2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'saved'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/numpy/lib/npyio.py\u001b[0m in \u001b[0;36msavetxt\u001b[0;34m(fname, X, fmt, delimiter, newline, header, footer, comments, encoding)\u001b[0m\n\u001b[1;32m   1373\u001b[0m                     raise TypeError(\"Mismatch between array dtype ('%s') and \"\n\u001b[1;32m   1374\u001b[0m                                     \u001b[0;34m\"format specifier ('%s')\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1375\u001b[0;31m                                     % (str(X.dtype), format))\n\u001b[0m\u001b[1;32m   1376\u001b[0m                 \u001b[0mfh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1377\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Mismatch between array dtype ('object') and format specifier ('%.18e')"
     ]
    }
   ],
   "source": [
    "print('saving tf matrix normalized...')\n",
    "np.savetxt('./representations/tf_matrix_normalized.txt', doc_term_matrix_l2)\n",
    "print('saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def numDocsContaining(word, doclist):\n",
    "    doccount = 0\n",
    "    for doc in doclist:\n",
    "        if freq(word, doc) > 0:\n",
    "            doccount +=1\n",
    "    return doccount \n",
    "\n",
    "def idf(word, doclist):\n",
    "    n_samples = len(doclist)\n",
    "    df = numDocsContaining(word, doclist)\n",
    "    return np.log(n_samples / 1+df)\n",
    "\n",
    "my_idf_vector = [idf(word, t) for word in voc]\n",
    "#print(my_idf_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_idf_matrix(idf_vector):\n",
    "    idf_mat = np.zeros((len(idf_vector), len(idf_vector)))\n",
    "    np.fill_diagonal(idf_mat, idf_vector)\n",
    "    return idf_mat\n",
    "\n",
    "my_idf_matrix = build_idf_matrix(my_idf_vector)\n",
    "#print(my_idf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_term_matrix_tfidf = []\n",
    "\n",
    "#performing tf-idf matrix multiplication\n",
    "for tf_vector in tf_matrix:\n",
    "    doc_term_matrix_tfidf.append(np.dot(tf_vector, my_idf_matrix))\n",
    "\n",
    "#normalizing\n",
    "doc_term_matrix_tfidf_l2 = []\n",
    "for tf_vector in doc_term_matrix_tfidf:\n",
    "    doc_term_matrix_tfidf_l2.append(l2_normalizer(tf_vector))\n",
    "                                    \n",
    "print(np.shape(np.matrix(doc_term_matrix_tfidf_l2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------- TF PARAMETERS ---------- #\n",
    "\n",
    "# Create a subplot with 1 row and 2 columns\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2)\n",
    "fig.set_size_inches(18, 7)\n",
    "\n",
    "h_tf = sorted(tf.items(), key=lambda x: x[1], reverse=True)\n",
    "ax1.plot( list(map(lambda x: x[1], h_tf)) )\n",
    "ax1.set_title('Tf Scores')\n",
    " \n",
    "ax2.plot( list(filter( lambda y: y >= 0.001 ,map(lambda x: x[1], h_tf)) )  )\n",
    "ax2.set_title('Tf Parameters : min 0.001 x max 0.004')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------- TFIDF PARAMETERS ---------- #\n",
    "\n",
    "# Create a subplot with 1 row and 2 columns\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2)\n",
    "fig.set_size_inches(18, 7)\n",
    "\n",
    "h_tfidf = sorted(tfidf.items(), key=lambda x: x[1], reverse=True)\n",
    "ax1.plot( list(map(lambda x: x[1], h_tfidf)) )\n",
    "ax1.set_title('TfIdf Scores')\n",
    " \n",
    "plt.plot( list(filter( lambda y: y > 0.0001 and y <= 0.001 ,map(lambda x: x[1], h_tfidf)) )  )\n",
    "ax2.set_title('TfIdf Parameters : min 0.0001 x max 0.001')\n",
    "plt.show()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}