Data_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 523,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 524,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import unidecode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 525,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_train = open('Penn_Corpus_sentc_tagged_train.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 526,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_train = []\n",
    "with open('Penn_Corpus_sentc_tagged_train.txt') as train_file:\n",
    "    file_train = train_file.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 527,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_test = []\n",
    "with open('Penn_Corpus_sentc_tagged_correct.txt') as test_file:\n",
    "    file_test = test_file.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 528,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##Splicing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 529,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_train = []\n",
    "for i in range(len(file_train)):\n",
    "    split_train.append(file_train[i].rsplit(' ._.', 1)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 530,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_test = []\n",
    "for i in range(len(file_test)):\n",
    "    split_test.append(file_test[i].rsplit(' ._.', 1)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 531,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Centerbank_NNP added_VBD 5\\\\/8_CD to_TO 8\\xa03\\\\/4_CD ;_: shares_NNS of_IN NESB_NNP ,_, a_DT New_NNP London-based_JJ bank_NN holding_VBG company_NN ,_, rose_VBD 5\\\\/8_CD to_TO 5\\xa07\\\\/8_CD'"
      ]
     },
     "execution_count": 531,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_train[900]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 532,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"If_IN ``_`` A_DT Wild_NNP Sheep_NNP Chase_NNP ''_'' carries_VBZ an_DT implicit_JJ message_NN for_IN international_JJ relations_NNS ,_, it_PRP 's_VBZ that_IN the_DT Japanese_NNS are_VBP more_RBR like_IN us_PRP than_IN most_JJS of_IN us_PRP think_VBP 0_CD-\""
      ]
     },
     "execution_count": 532,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_test[442].split(' ._.')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 533,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"If_IN ``_`` A_DT Wild_NNP Sheep_NNP Chase_NNP ''_'' carries_VBZ an_DT implicit_JJ message_NN for_IN international_JJ relations_NNS ,_, it_PRP 's_VBZ that_IN the_DT Japanese_NNPS are_VBP more_RBR like_IN us_PRP than_IN most_JJS of_IN us_PRP think_VB 0_CD\\n\""
      ]
     },
     "execution_count": 533,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_train[442].split(' ._.')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 534,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_test_2 = [w[:-1] if w.find('\\n') > 0 else w for w in split_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 535,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_train_2 = [w[:-2] if w.find(' \\n') > 0 else w for w in split_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(len(split_train_2)):\n",
    "    if(split_train_2[i].find(u'\\xa0') > 0):\n",
    "        split_train_2[i] = unidecode.unidecode(split_train_2[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 537,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Centerbank_NNP added_VBD 5\\\\/8_CD to_TO 8 3\\\\/4_CD ;_: shares_NNS of_IN NESB_NNP ,_, a_DT New_NNP London-based_JJ bank_NN holding_VBG company_NN ,_, rose_VBD 5\\\\/8_CD to_TO 5 7\\\\/8_CD'"
      ]
     },
     "execution_count": 537,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_train_2[900]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 538,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Centerbank_NNP added_VBD 5\\\\/8_CD to_TO 8 3\\\\/4_CD ;_: shares_NNS of_IN NESB_NNP ,_, a_DT New_NNP London-based_JJ bank_NN holding_VBG company_NN ,_, rose_VBD 5\\\\/8_CD to_TO 5 7\\\\/8_CD'"
      ]
     },
     "execution_count": 538,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 539,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##same length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "indx = []\n",
    "for i in range(len(file_train)):\n",
    "    if(split_train_2[i].count(' ') != split_test_2[i].count(' ')):\n",
    "        indx.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 541,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1036, 1800]"
      ]
     },
     "execution_count": 541,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 542,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(indx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'For_IN the_DT period_NN ended_VBN Sept._NNP 30_CD ,_, it_PRP earned_VBD 16.68_CD billion_CD yen_NNS ,_, -LRB-_-LRB- US$_$ 116.7_CD million_CD -RRB-_-RRB- up_RB from_IN 12.68_CD billion_CD yen_NNS the_DT year_NN before_RB'"
      ]
     },
     "execution_count": 543,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_train_2[3583]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'For_IN the_DT period_NN ended_VBD Sept._NNP 30_CD ,_, it_PRP earned_VBD 16.68_CD billion_CD yen_NN ,_, -LRB-_-LRB- US$_$ 116.7_CD million_CD -RRB-_-RRB- up_RB from_IN 12.68_CD billion_CD yen_NN the_DT year_NN before_IN'"
      ]
     },
     "execution_count": 544,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_test_2[3583]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##spliting sentence\n",
    "sent_split = file_train[41].split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#sent_ssplit = [a.split('_') for a in sent_split if a[0] != '*' or a[0].is]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 569,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## SENTENCE RAW ACCURACY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 570,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "for i in range(len(split_train_2)):\n",
    "    if(split_train_2[i] == split_test_2[i]):\n",
    "        count = count + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49974450689831373"
      ]
     },
     "execution_count": 549,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count/len(split_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 550,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## WORD COUNT ACCURACY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 561,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "words_train = [token for sentence in split_train_2 for token in sentence.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "words_test = [token for sentence in split_test_2 for token in sentence.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 563,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'and_CC'"
      ]
     },
     "execution_count": 563,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_train[51234]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 564,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'and_CC'"
      ]
     },
     "execution_count": 564,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_test[51234]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 565,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_2 = 0\n",
    "for i in range(len(words_train)):\n",
    "    if(words_train[i] == words_test[i]):\n",
    "        count_2 = count_2 + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 566,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.77840329218107"
      ]
     },
     "execution_count": 566,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_2 / len(words_train)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}