Day 86.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Natural Language Processing Core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.Tokenizing Words and Sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph = \"\"\"Tell General Howard I know his heart. What he told me before, I have it in my heart. I am tired of fighting.\n",
    "Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead. The old men are all dead. It is the young men who\n",
    "say yes or no. He who led on the young men is dead. It is cold, and we have no blankets; the little children are freezing \n",
    "to death. My people, some of them, have run away to the hills, and have no blankets, no food. No one knows where they are \n",
    "– perhaps freezing to death. I want to have time to look for my children, and see how many of them I can find. Maybe I \n",
    "shall find them among the dead. Hear me, my Chiefs! I am tired; my heart is sick and sad. From where the sun now stands \n",
    "I will fight no more forever.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tell General Howard I know his heart.',\n",
       " 'What he told me before, I have it in my heart.',\n",
       " 'I am tired of fighting.',\n",
       " 'Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead.',\n",
       " 'The old men are all dead.',\n",
       " 'It is the young men who\\nsay yes or no.',\n",
       " 'He who led on the young men is dead.',\n",
       " 'It is cold, and we have no blankets; the little children are freezing \\nto death.',\n",
       " 'My people, some of them, have run away to the hills, and have no blankets, no food.',\n",
       " 'No one knows where they are \\n– perhaps freezing to death.',\n",
       " 'I want to have time to look for my children, and see how many of them I can find.',\n",
       " 'Maybe I \\nshall find them among the dead.',\n",
       " 'Hear me, my Chiefs!',\n",
       " 'I am tired; my heart is sick and sad.',\n",
       " 'From where the sun now stands \\nI will fight no more forever.']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = nltk.sent_tokenize(paragraph)   # seperate each sentences of our paragraph\n",
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = nltk.word_tokenize(paragraph)       # seperate each words of sentences of our paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tell',\n",
       " 'General',\n",
       " 'Howard',\n",
       " 'I',\n",
       " 'know',\n",
       " 'his',\n",
       " 'heart',\n",
       " '.',\n",
       " 'What',\n",
       " 'he',\n",
       " 'told',\n",
       " 'me',\n",
       " 'before',\n",
       " ',',\n",
       " 'I',\n",
       " 'have',\n",
       " 'it',\n",
       " 'in',\n",
       " 'my',\n",
       " 'heart',\n",
       " '.',\n",
       " 'I',\n",
       " 'am',\n",
       " 'tired',\n",
       " 'of',\n",
       " 'fighting',\n",
       " '.',\n",
       " 'Our',\n",
       " 'Chiefs',\n",
       " 'are',\n",
       " 'killed',\n",
       " ';',\n",
       " 'Looking',\n",
       " 'Glass',\n",
       " 'is',\n",
       " 'dead',\n",
       " ',',\n",
       " 'Ta',\n",
       " 'Hool',\n",
       " 'Hool',\n",
       " 'Shute',\n",
       " 'is',\n",
       " 'dead',\n",
       " '.',\n",
       " 'The',\n",
       " 'old',\n",
       " 'men',\n",
       " 'are',\n",
       " 'all',\n",
       " 'dead',\n",
       " '.',\n",
       " 'It',\n",
       " 'is',\n",
       " 'the',\n",
       " 'young',\n",
       " 'men',\n",
       " 'who',\n",
       " 'say',\n",
       " 'yes',\n",
       " 'or',\n",
       " 'no',\n",
       " '.',\n",
       " 'He',\n",
       " 'who',\n",
       " 'led',\n",
       " 'on',\n",
       " 'the',\n",
       " 'young',\n",
       " 'men',\n",
       " 'is',\n",
       " 'dead',\n",
       " '.',\n",
       " 'It',\n",
       " 'is',\n",
       " 'cold',\n",
       " ',',\n",
       " 'and',\n",
       " 'we',\n",
       " 'have',\n",
       " 'no',\n",
       " 'blankets',\n",
       " ';',\n",
       " 'the',\n",
       " 'little',\n",
       " 'children',\n",
       " 'are',\n",
       " 'freezing',\n",
       " 'to',\n",
       " 'death',\n",
       " '.',\n",
       " 'My',\n",
       " 'people',\n",
       " ',',\n",
       " 'some',\n",
       " 'of',\n",
       " 'them',\n",
       " ',',\n",
       " 'have',\n",
       " 'run',\n",
       " 'away',\n",
       " 'to',\n",
       " 'the',\n",
       " 'hills',\n",
       " ',',\n",
       " 'and',\n",
       " 'have',\n",
       " 'no',\n",
       " 'blankets',\n",
       " ',',\n",
       " 'no',\n",
       " 'food',\n",
       " '.',\n",
       " 'No',\n",
       " 'one',\n",
       " 'knows',\n",
       " 'where',\n",
       " 'they',\n",
       " 'are',\n",
       " '–',\n",
       " 'perhaps',\n",
       " 'freezing',\n",
       " 'to',\n",
       " 'death',\n",
       " '.',\n",
       " 'I',\n",
       " 'want',\n",
       " 'to',\n",
       " 'have',\n",
       " 'time',\n",
       " 'to',\n",
       " 'look',\n",
       " 'for',\n",
       " 'my',\n",
       " 'children',\n",
       " ',',\n",
       " 'and',\n",
       " 'see',\n",
       " 'how',\n",
       " 'many',\n",
       " 'of',\n",
       " 'them',\n",
       " 'I',\n",
       " 'can',\n",
       " 'find',\n",
       " '.',\n",
       " 'Maybe',\n",
       " 'I',\n",
       " 'shall',\n",
       " 'find',\n",
       " 'them',\n",
       " 'among',\n",
       " 'the',\n",
       " 'dead',\n",
       " '.',\n",
       " 'Hear',\n",
       " 'me',\n",
       " ',',\n",
       " 'my',\n",
       " 'Chiefs',\n",
       " '!',\n",
       " 'I',\n",
       " 'am',\n",
       " 'tired',\n",
       " ';',\n",
       " 'my',\n",
       " 'heart',\n",
       " 'is',\n",
       " 'sick',\n",
       " 'and',\n",
       " 'sad',\n",
       " '.',\n",
       " 'From',\n",
       " 'where',\n",
       " 'the',\n",
       " 'sun',\n",
       " 'now',\n",
       " 'stands',\n",
       " 'I',\n",
       " 'will',\n",
       " 'fight',\n",
       " 'no',\n",
       " 'more',\n",
       " 'forever',\n",
       " '.']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.Stemming\n",
    "\"Stemming is process of reducing infected or derived words to their word stem, base or root form\"\n",
    "###### Words representation may not have any meaning.\n",
    "##### Takes less time\n",
    "##### Use stemming when meaning of words are not important for analysis. example spam detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stemming from nltk\n",
    "from nltk.stem import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init the stemming\n",
    "stemmer = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tell gener howard I know hi heart .',\n",
       " 'what he told me befor , I have it in my heart .',\n",
       " 'I am tire of fight .',\n",
       " 'our chief are kill ; look glass is dead , Ta hool hool shute is dead .',\n",
       " 'the old men are all dead .',\n",
       " 'It is the young men who say ye or no .',\n",
       " 'He who led on the young men is dead .',\n",
       " 'It is cold , and we have no blanket ; the littl children are freez to death .',\n",
       " 'My peopl , some of them , have run away to the hill , and have no blanket , no food .',\n",
       " 'No one know where they are – perhap freez to death .',\n",
       " 'I want to have time to look for my children , and see how mani of them I can find .',\n",
       " 'mayb I shall find them among the dead .',\n",
       " 'hear me , my chief !',\n",
       " 'I am tire ; my heart is sick and sad .',\n",
       " 'from where the sun now stand I will fight no more forev .']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#stemming\n",
    "for i in range(len(sentences)):\n",
    "    words = nltk.word_tokenize(sentences[i])\n",
    "    newwords = [stemmer.stem(word) for word in words]\n",
    "    sentences[i]= ' '.join(newwords)\n",
    "    \n",
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.Lemmatization\n",
    "\"Same as Stemming but intermediate representation/root form has a meaning\"\n",
    "###### Words representation  have  meaning.\n",
    "##### Takes more time than Stemming\n",
    "##### Use Lemmatization when meaning of words are  important for analysis. example Question answer application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import Lemmatization from NLTK\n",
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = nltk.sent_tokenize(paragraph)   # seperate each sentences of our paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init the Lemmatization model\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['TellGeneralHowardIknowhisheart.',\n",
       " 'Whathetoldmebefore,Ihaveitinmyheart.',\n",
       " 'Iamtiredoffighting.',\n",
       " 'OurChiefsarekilled;LookingGlassisdead,TaHoolHoolShuteisdead.',\n",
       " 'Theoldmenarealldead.',\n",
       " 'Itistheyoungmenwhosayyesorno.',\n",
       " 'Hewholedontheyoungmenisdead.',\n",
       " 'Itiscold,andwehavenoblanket;thelittlechildarefreezingtodeath.',\n",
       " 'Mypeople,someofthem,haverunawaytothehill,andhavenoblanket,nofood.',\n",
       " 'Nooneknowwheretheyare–perhapsfreezingtodeath.',\n",
       " 'Iwanttohavetimetolookformychild,andseehowmanyofthemIcanfind.',\n",
       " 'MaybeIshallfindthemamongthedead.',\n",
       " 'Hearme,myChiefs!',\n",
       " 'Iamtired;myheartissickandsad.',\n",
       " 'FromwherethesunnowstandIwillfightnomoreforever.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lemmatization\n",
    "for i in range(len(sentences)):\n",
    "    words = nltk.word_tokenize(sentences[i])\n",
    "    newwords = [lemmatizer.lemmatize(word) for word in words]\n",
    "    sentences[i] = ''.join(newwords)\n",
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.Stop Word Removal using nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = nltk.sent_tokenize(paragraph)   # seperate each sentences of our paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tell General Howard I know heart .',\n",
       " 'What told , I heart .',\n",
       " 'I tired fighting .',\n",
       " 'Our Chiefs killed ; Looking Glass dead , Ta Hool Hool Shute dead .',\n",
       " 'The old men dead .',\n",
       " 'It young men say yes .',\n",
       " 'He led young men dead .',\n",
       " 'It cold , blankets ; little children freezing death .',\n",
       " 'My people , , run away hills , blankets , food .',\n",
       " 'No one knows – perhaps freezing death .',\n",
       " 'I want time look children , see many I find .',\n",
       " 'Maybe I shall find among dead .',\n",
       " 'Hear , Chiefs !',\n",
       " 'I tired ; heart sick sad .',\n",
       " 'From sun stands I fight forever .']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stop word removal\n",
    "for i in range(len(sentences)):\n",
    "    words = nltk.word_tokenize(sentences[i])\n",
    "    newwords = [word for word in words if word not in stopwords.words('english')]\n",
    "    sentences[i] = ' '.join(newwords)\n",
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.Parts of Speech Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph = 'hi! how are you. where from you. '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = nltk.word_tokenize(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagged_words =nltk.pos_tag(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hi_NN !_. how_WRB are_VBP you_PRP ._. where_WRB from_IN you_PRP ._.'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tags = []\n",
    "for tw in tagged_words:\n",
    "    word_tags.append(tw[0]+\"_\"+tw[1])\n",
    "    \n",
    "tagged_paragraph = ' '.join(word_tags)\n",
    "tagged_paragraph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.Named Entity Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph1 = \"Fouder of Wavy AI Research Foundation is from Pakistan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Fouder',\n",
       " 'of',\n",
       " 'Wavy',\n",
       " 'AI',\n",
       " 'Research',\n",
       " 'Foundation',\n",
       " 'is',\n",
       " 'from',\n",
       " 'Pakistan']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = nltk.word_tokenize(paragraph1)\n",
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Fouder', 'NN'),\n",
       " ('of', 'IN'),\n",
       " ('Wavy', 'NNP'),\n",
       " ('AI', 'NNP'),\n",
       " ('Research', 'NNP'),\n",
       " ('Foundation', 'NNP'),\n",
       " ('is', 'VBZ'),\n",
       " ('from', 'IN'),\n",
       " ('Pakistan', 'NNP')]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_words = nltk.pos_tag(words)\n",
    "tagged_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "namedEnt = nltk.ne_chunk(tagged_words)\n",
    "namedEnt.draw()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.Building a Bags of Word Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import nltk\n",
    "import re\n",
    "import heapq\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph = \"\"\"Tell General Howard I know his heart. What he told me before, I have it in my heart. I am tired of fighting.\n",
    "Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead. The old men are all dead. It is the young men who\n",
    "say yes or no. He who led on the young men is dead. It is cold, and we have no blankets; the little children are freezing \n",
    "to death. My people, some of them, have run away to the hills, and have no blankets, no food. No one knows where they are \n",
    "– perhaps freezing to death. I want to have time to look for my children, and see how many of them I can find. Maybe I \n",
    "shall find them among the dead. Hear me, my Chiefs! I am tired; my heart is sick and sad. From where the sun now stands \n",
    "I will fight no more forever.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset= nltk.sent_tokenize(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Clean the text\n",
    "for i in range(len(dataset)):\n",
    "    dataset[i] = dataset[i].lower()\n",
    "    dataset[i] = re.sub(r'\\W',' ', dataset[i])\n",
    "    dataset[i] = re.sub(r'\\s+',' ', dataset[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the histogram\n",
    "word2count = {}\n",
    "for data in dataset:\n",
    "    words = nltk.word_tokenize(data)\n",
    "    for word in words:\n",
    "        if word not in word2count.keys():\n",
    "            word2count[word] = 1\n",
    "        else:\n",
    "            word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a 100 most frequent words from above dictionaries\n",
    "freq_words = heapq.nlargest(100, word2count, key= word2count.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finally building our BOW model\n",
    "X = []\n",
    "for data in dataset:\n",
    "    vector = []\n",
    "    for word in freq_words:\n",
    "        if word in nltk.word_tokenize(data):\n",
    "            vector.append(1)\n",
    "        else:\n",
    "            vector.append(0)\n",
    "    X.append(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, ..., 0, 0, 0],\n",
       "       [1, 0, 0, ..., 0, 0, 0],\n",
       "       [1, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [1, 0, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 0, ..., 1, 1, 1]])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Bag_of_Word_Model = np.asarray(X)\n",
    "Bag_of_Word_Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8.Building the TF-IDF Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import nltk\n",
    "import re\n",
    "import heapq\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph = \"\"\"Tell General Howard I know his heart. What he told me before, I have it in my heart. I am tired of fighting.\n",
    "Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead. The old men are all dead. It is the young men who\n",
    "say yes or no. He who led on the young men is dead. It is cold, and we have no blankets; the little children are freezing \n",
    "to death. My people, some of them, have run away to the hills, and have no blankets, no food. No one knows where they are \n",
    "– perhaps freezing to death. I want to have time to look for my children, and see how many of them I can find. Maybe I \n",
    "shall find them among the dead. Hear me, my Chiefs! I am tired; my heart is sick and sad. From where the sun now stands \n",
    "I will fight no more forever.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset= nltk.sent_tokenize(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Clean the text\n",
    "for i in range(len(dataset)):\n",
    "    dataset[i] = dataset[i].lower()\n",
    "    dataset[i] = re.sub(r'\\W',' ', dataset[i])\n",
    "    dataset[i] = re.sub(r'\\s+',' ', dataset[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the histogram\n",
    "word2count = {}\n",
    "for data in dataset:\n",
    "    words = nltk.word_tokenize(data)\n",
    "    for word in words:\n",
    "        if word not in word2count.keys():\n",
    "            word2count[word] = 1\n",
    "        else:\n",
    "            word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a 100 most frequent words from above dictionaries\n",
    "freq_words = heapq.nlargest(100, word2count, key= word2count.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IDF Matrix\n",
    "word_idfs = {}\n",
    "for word in  freq_words:\n",
    "    doc_count = 0\n",
    "    for data in dataset:\n",
    "        if word in nltk.word_tokenize(data):\n",
    "            doc_count += 1\n",
    "    word_idfs[word] = np.log((len(dataset)/doc_count)+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF Matrix\n",
    "tf_matrix = {}\n",
    "for word in freq_words:\n",
    "    doc_tf = []\n",
    "    for data in dataset:\n",
    "        frequency = 0\n",
    "        for w in nltk.word_tokenize(data):\n",
    "            if w== word:\n",
    "                frequency += 1\n",
    "        tf_word = frequency/len(nltk.word_tokenize(data))\n",
    "        doc_tf.append(tf_word)\n",
    "    tf_matrix[word] = doc_tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF-IDF Calculation\n",
    "tfidf_matrix = []\n",
    "for word in tf_matrix.keys():\n",
    "    tfidf = []\n",
    "    for value in tf_matrix[word]:\n",
    "        score = value * word_idfs[word]\n",
    "        tfidf.append(score)\n",
    "    tfidf_matrix.append(tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.asarray(tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.16359033, 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.10410294, 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.22902646, 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.12723692, 0.        , 0.15403271, ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.09542769, 0.09542769, 0.        , ..., 0.23104906, 0.23104906,\n",
       "        0.23104906]])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TF_IDF_Model = np.transpose(X)\n",
    "TF_IDF_Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.Building the N-Gram Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. N-Gram Modeling - Character Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data\n",
    "text= \"\"\"There are many different types of essay writing and, depending on what you are writing for, the format and approach \n",
    "can change. We’ve designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in \n",
    "the right direction – no matter if your essay is designed to persuade the college admissions team that you’re the right \n",
    "candidate, if you are making your case to win a scholarship, or simply if you are looking for help with your homework There are\n",
    "many different types of essay writing and, depending on what you are writing for, the format and approach can change. We’ve \n",
    "designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in the right direction \n",
    "– no matter if your essay is designed to persuade the college admissions team that you’re the right candidate, if you are \n",
    "making your case to win a scholarship, or simply if you are looking for help with your homework\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating the n-gram\n",
    "n = 3\n",
    "ngrams = {}\n",
    "for i in range(len(text)-n):\n",
    "    gram = text[i:i+n]  #text[0:3] = The\n",
    "    if gram not in ngrams.keys():\n",
    "        ngrams[gram] = []\n",
    "    ngrams[gram].append(text[i+n]) #text[0+3] = text[3] = r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There looking for, the right \n",
      "can change. We’ve designed in a scholarship, or simply is dependidate, if\n"
     ]
    }
   ],
   "source": [
    "# Testing our n-gram model\n",
    "currentGram = text[0:n]\n",
    "result = currentGram\n",
    "for i in range(100):\n",
    "    if currentGram not in ngrams.keys():\n",
    "        break\n",
    "    possibilities = ngrams[currentGram]\n",
    "    nextItem = possibilities[random.randrange(len(possibilities))]\n",
    "    result += nextItem\n",
    "    currentGram = result[len(result)-n: len(result)]\n",
    "    \n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. N-Gram Modeling - Word Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import librararies\n",
    "import random\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample data\n",
    "text= \"\"\"There are many different types of essay writing and, depending on what you are writing for, the format and approach \n",
    "can change. We’ve designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in \n",
    "the right direction – no matter if your essay is designed to persuade the college admissions team that you’re the right \n",
    "candidate, if you are making your case to win a scholarship, or simply if you are looking for help with your homework There are\n",
    "many different types of essay writing and, depending on what you are writing for, the format and approach can change. We’ve \n",
    "designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in the right direction \n",
    "– no matter if your essay is designed to persuade the college admissions team that you’re the right candidate, if you are \n",
    "making your case to win a scholarship, or simply if you are looking for help with your homework\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating the n-gram\n",
    "n = 3\n",
    "ngrams = {}\n",
    "words = nltk.word_tokenize(text)\n",
    "for i in range(len(words)-n):\n",
    "    gram = ' '.join(words[i:i+n])\n",
    "    if gram not in ngrams.keys():\n",
    "        ngrams[gram] = []\n",
    "    ngrams[gram].append(words[i+n]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are many different types of essay writing and , depending on what you are writing for , the format and approach can change . We ’ ve designed our Essay Writing Center\n"
     ]
    }
   ],
   "source": [
    "# Testing our n-gram model\n",
    "currentGram = ' '.join(words[0:n])\n",
    "result = currentGram\n",
    "for i in range(30):\n",
    "    if currentGram not in ngrams.keys():\n",
    "        break\n",
    "    possibilities = ngrams[currentGram]\n",
    "    nextItem = possibilities[random.randrange(len(possibilities))]\n",
    "    result += ' '+nextItem\n",
    "    rwords = nltk.word_tokenize(result)\n",
    "    currentGram = ' '.join(rwords[len(rwords)-n:len(rwords)])\n",
    "    \n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10.Latent Semantic Analysis using Scikit Learn and nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the Libraries\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample data\n",
    "dataset = [\"The amount of population is increase day by day\",\n",
    "           \"The concert was just a great\",\n",
    "          \"I love to see Gordon Ramsay cook\",\n",
    "           \"Google introducing a new technology\",\n",
    "          \"AI Robot are example of AI technology present today\",\n",
    "          \"All of us singing in the concert\",\n",
    "          \"We have launch compaigns to stop popullation and Global warming\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert into lowercase\n",
    "dataset = [line.lower() for line in dataset]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert into TF-IDF model\n",
    "vectorizer = TfidfVectorizer()\n",
    "X = vectorizer.fit_transform(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TruncatedSVD(algorithm='randomized', n_components=4, n_iter=100,\n",
       "       random_state=None, tol=0.0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lSA\n",
    "lsa = TruncatedSVD(n_components = 4, n_iter = 100)\n",
    "lsa.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "row1 = lsa.components_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Concept 0 :\n",
      "('the', 0.4259712396390564)\n",
      "('concert', 0.39998286562925545)\n",
      "('of', 0.2938250158823831)\n",
      "('great', 0.24619048735802163)\n",
      "('just', 0.24619048735802163)\n",
      "('was', 0.24619048735802163)\n",
      "('day', 0.23699958677792862)\n",
      "('all', 0.23566672236585795)\n",
      "('in', 0.23566672236585795)\n",
      "('singing', 0.23566672236585795)\n",
      "\n",
      "Concept 1 :\n",
      "('technology', 0.4518449046133094)\n",
      "('ai', 0.3979413369682742)\n",
      "('google', 0.3453644610279633)\n",
      "('introducing', 0.3453644610279633)\n",
      "('new', 0.3453644610279633)\n",
      "('are', 0.19897066848413705)\n",
      "('example', 0.19897066848413705)\n",
      "('present', 0.19897066848413705)\n",
      "('robot', 0.19897066848413705)\n",
      "('today', 0.19897066848413705)\n",
      "\n",
      "Concept 2 :\n",
      "('to', 0.4157884439670068)\n",
      "('cook', 0.2835916579351077)\n",
      "('gordon', 0.2835916579351077)\n",
      "('love', 0.2835916579351077)\n",
      "('ramsay', 0.2835916579351077)\n",
      "('see', 0.2835916579351077)\n",
      "('and', 0.2173064471129242)\n",
      "('compaigns', 0.2173064471129242)\n",
      "('global', 0.2173064471129242)\n",
      "('have', 0.2173064471129242)\n",
      "\n",
      "Concept 3 :\n",
      "('day', 0.5130530624437486)\n",
      "('amount', 0.2565265312218743)\n",
      "('by', 0.2565265312218743)\n",
      "('increase', 0.2565265312218743)\n",
      "('is', 0.2565265312218743)\n",
      "('population', 0.2565265312218743)\n",
      "('of', 0.17865460887270906)\n",
      "('ai', 0.11695203723130627)\n",
      "('are', 0.058476018615653116)\n",
      "('example', 0.058476018615653116)\n"
     ]
    }
   ],
   "source": [
    "terms = vectorizer.get_feature_names()\n",
    "for i, comp in enumerate(lsa.components_):\n",
    "    componentTerms = zip(terms, comp)\n",
    "    sortedTerms = sorted(componentTerms, key=lambda x:x[1], reverse=True)\n",
    "    sortedTerms = sortedTerms[:10]\n",
    "    print(\"\\nConcept\",i, \":\")\n",
    "    for term in sortedTerms:\n",
    "        print(term)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concept word\n",
    "concept_words = {}\n",
    "terms = vectorizer.get_feature_names()\n",
    "for i, comp in enumerate(lsa.components_):\n",
    "    componentTerms = zip(terms, comp)\n",
    "    sortedTerms = sorted(componentTerms, key=lambda x:x[1], reverse=True)\n",
    "    sortedTerms = sortedTerms[:10]\n",
    "    concept_words[\"Concept \"+str(i)] = sortedTerms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Concept 0:\n",
      "1.1937954290772967\n",
      "1.5645255673423768\n",
      "0\n",
      "0\n",
      "0.2938250158823831\n",
      "1.8267792882482687\n",
      "0\n",
      "\n",
      "Concept 1:\n",
      "0\n",
      "0\n",
      "0\n",
      "1.4879382876971992\n",
      "2.242580920970543\n",
      "0\n",
      "0\n",
      "\n",
      "Concept 2:\n",
      "0\n",
      "0\n",
      "1.8337467336425453\n",
      "0\n",
      "0\n",
      "0\n",
      "1.2850142324187037\n",
      "\n",
      "Concept 3:\n",
      "2.4873933898695775\n",
      "0\n",
      "0\n",
      "0\n",
      "0.5295107205666278\n",
      "0.17865460887270906\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "#now concept are belong from which categories take the probablities\n",
    "for key in concept_words.keys():\n",
    "    sentence_score = []\n",
    "    for sentence in dataset:\n",
    "        words= nltk.word_tokenize(sentence)\n",
    "        score = 0\n",
    "        for word in words:\n",
    "            for word_with_score in concept_words[key]:\n",
    "                if word == word_with_score[0]:\n",
    "                    score += word_with_score[1]\n",
    "        sentence_score.append(score)\n",
    "    print(\"\\n\"+key+\":\")\n",
    "    for sentence_scores in sentence_score:\n",
    "        print(sentence_scores)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}