sentimental_analysis_Model_ASTD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Project Title :  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Folder:\n",
    "Data/tweet.txt\n",
    "Data/tweet_x.csv\n",
    "\n",
    "Libraries:\n",
    "sklearn\n",
    "scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "from Definations import *\n",
    "from Utilities import *\n",
    "from sklearn.feature_selection.univariate_selection import SelectPercentile\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.manifold import isomap\n",
    "import numpy as np\n",
    "from scipy.sparse import hstack\n",
    "\n",
    "\n",
    "\n",
    "# TODO: Import the three supervised learning models from sklearn\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import make_classification\n",
    "\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import fbeta_score\n",
    "\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'clf': LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "            intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "            penalty='l2', random_state=0, solver='liblinear', tol=0.0001,\n",
       "            verbose=0, warm_start=False),\n",
       "  'name': 'Logistic Regression',\n",
       "  'parameter_tunning': True,\n",
       "  'tune_clf': GridSearchCV(cv=3, error_score='raise',\n",
       "         estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "            intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "            penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "            verbose=0, warm_start=False),\n",
       "         fit_params={}, iid=True, n_jobs=1,\n",
       "         param_grid=[{'penalty': ['hinge'], 'C': [1.0, 2.7825594, 7.74263683, 21.5443469, 59.948425, 166.810054, 464.158883, 1291.54967, 3593.81366, 10000.0]}],\n",
       "         pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)},\n",
       " {'clf': PassiveAggressiveClassifier(C=1.0, class_weight=None, fit_intercept=True,\n",
       "                loss='hinge', n_iter=5, n_jobs=1, random_state=0,\n",
       "                shuffle=True, verbose=0, warm_start=False),\n",
       "  'name': 'Passive Aggresive',\n",
       "  'parameter_tunning': True,\n",
       "  'tune_clf': GridSearchCV(cv=None, error_score='raise',\n",
       "         estimator=PassiveAggressiveClassifier(C=1.0, class_weight=None, fit_intercept=True,\n",
       "                loss='hinge', n_iter=5, n_jobs=1, random_state=None,\n",
       "                shuffle=True, verbose=0, warm_start=False),\n",
       "         fit_params={}, iid=True, n_jobs=1,\n",
       "         param_grid={'warm_start': [False, True], 'loss': ['hinge'], 'C': [1.0, 0.7, 0.5, 0.3, 0.1], 'n_jobs': [1], 'shuffle': [True, False], 'verbose': [0], 'n_iter': [15], 'random_state': [0]},\n",
       "         pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)},\n",
       " {'clf': LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "       intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "       multi_class='ovr', penalty='l2', random_state=0, tol=0.001, verbose=0),\n",
       "  'name': 'SVM',\n",
       "  'parameter_tunning': True,\n",
       "  'tune_clf': GridSearchCV(cv=None, error_score='raise',\n",
       "         estimator=LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "       intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "       multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "       verbose=0),\n",
       "         fit_params={}, iid=True, n_jobs=1,\n",
       "         param_grid={'loss': ['hinge', 'squared_hinge'], 'C': [1.0, 0.7, 0.5, 0.3, 0.1], 'random_state': [0], 'intercept_scaling': [0.3, 0.5, 1], 'dual': [True], 'fit_intercept': [True, False], 'max_iter': [1000], 'multi_class': ['ovr', 'crammer_singer']},\n",
       "         pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)},\n",
       " {'clf': Perceptron(alpha=0.0001, class_weight=None, eta0=1.0, fit_intercept=True,\n",
       "        n_iter=5, n_jobs=1, penalty=None, random_state=0, shuffle=True,\n",
       "        verbose=0, warm_start=False),\n",
       "  'name': 'Perceptron',\n",
       "  'parameter_tunning': True,\n",
       "  'tune_clf': GridSearchCV(cv=None, error_score='raise',\n",
       "         estimator=Perceptron(alpha=0.0001, class_weight=None, eta0=1.0, fit_intercept=True,\n",
       "        n_iter=5, n_jobs=1, penalty=None, random_state=0, shuffle=True,\n",
       "        verbose=0, warm_start=False),\n",
       "         fit_params={}, iid=True, n_jobs=1,\n",
       "         param_grid={'penalty': ['l1', 'l2'], 'n_iter': [5, 10, 15], 'random_state': [0], 'shuffle': [True, False], 'eta0': [1.0, 0.5], 'fit_intercept': [True, False], 'alpha': [1, 0.8, 0.5, 0.3, 0.1, 0.01, 0.001, 0.0001], 'warm_start': [True, False]},\n",
       "         pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)},\n",
       " {'clf': BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True),\n",
       "  'name': 'bnb',\n",
       "  'parameter_tunning': True,\n",
       "  'tune_clf': GridSearchCV(cv=None, error_score='raise',\n",
       "         estimator=BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True),\n",
       "         fit_params={}, iid=True, n_jobs=1,\n",
       "         param_grid={'binarize': [0.0, 0.25, 0.5], 'alpha': [1.0, 0.8, 0.6, 0.4, 0.2], 'fit_prior': [True, False]},\n",
       "         pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)},\n",
       " {'clf': SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "         eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "         learning_rate='optimal', loss='hinge', n_iter=5, n_jobs=1,\n",
       "         penalty='l2', power_t=0.5, random_state=0, shuffle=True, verbose=0,\n",
       "         warm_start=False), 'name': 'sgd', 'parameter_tunning': False},\n",
       " {'clf': KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='euclidean',\n",
       "             metric_params={'random_state': 0}, n_jobs=1, n_neighbors=5, p=2,\n",
       "             weights='uniform'),\n",
       "  'name': 'KNN',\n",
       "  'parameter_tunning': False,\n",
       "  'tune_clf': GridSearchCV(cv=5, error_score='raise',\n",
       "         estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "             metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "             weights='uniform'),\n",
       "         fit_params={}, iid=True, n_jobs=1,\n",
       "         param_grid=[{'n_neighbors': [5, 10, 50, 100], 'metric': ['euclidean', 'minkowski'], 'p': [2, 3, 4, 5]}],\n",
       "         pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)},\n",
       " {'clf': RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "              max_depth=110, max_features='auto', max_leaf_nodes=None,\n",
       "              min_samples_leaf=2, min_samples_split=20,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "              oob_score=False, random_state=0, verbose=0, warm_start=False),\n",
       "  'name': 'RandomForest',\n",
       "  'parameter_tunning': False},\n",
       " {'clf': DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "              max_features=None, max_leaf_nodes=None, min_samples_leaf=1,\n",
       "              min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "              presort=False, random_state=None, splitter='best'),\n",
       "  'name': 'DecisionTree',\n",
       "  'parameter_tunning': False},\n",
       " {'clf': MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True),\n",
       "  'name': 'mnb',\n",
       "  'parameter_tunning': True,\n",
       "  'tune_clf': GridSearchCV(cv=5, error_score='raise',\n",
       "         estimator=MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True),\n",
       "         fit_params={}, iid=True, n_jobs=1,\n",
       "         param_grid={'alpha': [1.0, 0.1, 0.3, 0.6, 0.8, 0.0], 'fit_prior': [True, False]},\n",
       "         pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)}]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifiers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First Stage :  Dataset Exploration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step _: Benchmark Experiment Replication using ASTD. ###"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. It stands for Arabic Sentemental Tweet Dataset which is used for arabic social sentimental analysis .\n",
    "2. Consist of 10000 tweets gathered from twitter.\n",
    "3. Each tweet is Classified as: objective, subjective positive, subjective negative or subjective mixed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step _: Loading the data set in jupyter for exploration. ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import libraries necessary for this project\n",
    "import pandas as pd\n",
    "from time import time\n",
    "from IPython.display import display # Allows the use of display() for DataFrames\n",
    "\n",
    "# Import supplementary visualization code visuals.py\n",
    "import visuals as vs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Text File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "بعد استقالة رئيس #المحكمة_الدستورية ننتظر استقالة #رئيس_القضاء #السودان\tOBJ\r\n",
      "\n",
      "أهنئ الدكتور أحمد جمال الدين، القيادي بحزب مصر، بمناسبة صدور أولى روايته\tPOS\r\n",
      "\n",
      "البرادعي يستقوى بامريكا مرةاخرى و يرسل عصام العريان الي واشنطن شئ مقرف\tNEG\r\n",
      "\n",
      "#الحرية_والعدالة | شاهد الآن: #ليلة_الاتحادية أول فيلم استقصائي يتناول أسرار و كواليس تعرض لأول مرة حول حقيقة\tOBJ\r\n",
      "\n",
      "الوالدة لو اقولها بخاطري حشيشة تضحك بس من اقولها ملل الله وكيلك تعطيني محاضرة عن الفسق والفجور بجنوب الشيشان #ليه كذا يانبع الحنان\tNEUTRAL\r\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import codecs\n",
    "filename = codecs.open('data\\Tweets.txt', 'r', encoding=\"utf-8\")\n",
    "outputfile = filename.readlines()\n",
    "for line in outputfile[0:5]:\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### CSV File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>Classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>بعد استقالة رئيس #المحكمة_الدستورية ننتظر استق...</td>\n",
       "      <td>OBJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>أهنئ الدكتور أحمد جمال الدين، القيادي بحزب مصر...</td>\n",
       "      <td>POS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>البرادعي يستقوى بامريكا مرةاخرى و يرسل عصام ال...</td>\n",
       "      <td>NEG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>#الحرية_والعدالة | شاهد الآن: #ليلة_الاتحادية ...</td>\n",
       "      <td>OBJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>الوالدة لو اقولها بخاطري حشيشة تضحك بس من اقول...</td>\n",
       "      <td>NEUTRAL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Tweet Classification\n",
       "0  بعد استقالة رئيس #المحكمة_الدستورية ننتظر استق...            OBJ\n",
       "1  أهنئ الدكتور أحمد جمال الدين، القيادي بحزب مصر...            POS\n",
       "2  البرادعي يستقوى بامريكا مرةاخرى و يرسل عصام ال...            NEG\n",
       "3  #الحرية_والعدالة | شاهد الآن: #ليلة_الاتحادية ...            OBJ\n",
       "4  الوالدة لو اقولها بخاطري حشيشة تضحك بس من اقول...        NEUTRAL"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data\\Tweets_x.csv', delimiter='\\t',names= [\"Tweet\", \"Classification\"])\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>Classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10006</td>\n",
       "      <td>9986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>10002</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>يوسف الحسيني بالفيديو يفضح كذب قناة الجزيرة وي...</td>\n",
       "      <td>OBJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>2</td>\n",
       "      <td>6675</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Tweet Classification\n",
       "count                                               10006           9986\n",
       "unique                                              10002              4\n",
       "top     يوسف الحسيني بالفيديو يفضح كذب قناة الجزيرة وي...            OBJ\n",
       "freq                                                    2           6675"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AS it can be clearly seen, I lost some data in this process since i converted the txt into csv using excel. I removed them to keep dataset consistant but I will fix this issue later"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Show Count of All Categories ####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>Classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>9986</td>\n",
       "      <td>9986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>9982</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>الإخوان يطلقون طفايات الحريق للإيحاء بإلقاء ال...</td>\n",
       "      <td>OBJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>2</td>\n",
       "      <td>6675</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Tweet Classification\n",
       "count                                                9986           9986\n",
       "unique                                               9982              4\n",
       "top     الإخوان يطلقون طفايات الحريق للإيحاء بإلقاء ال...            OBJ\n",
       "freq                                                    2           6675"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OBJ        6675\n",
      "NEG        1682\n",
      "NEUTRAL     831\n",
      "POS         798\n",
      "Name: Classification, dtype: int64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAYAAAC926RRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFpBJREFUeJzt3X+w3XV95/HnSyK1UpFEY4ZNYmFrKoPjqHgH6Nb+WDOG\ngJ2GtcrCuJJls5vuTNr1184WO7NllDpLtcpqd8tMVtKGjhUZ20q2MtIMYt2drkJQigV0cqVgkgKJ\nJmItFQu+94/zuXqIudxzkpNzcs/3+Zi5cz7fz/dzzn1/z9w5r/P9fj/f701VIUnqnmdNugBJ0mQY\nAJLUUQaAJHWUASBJHWUASFJHGQCS1FELBkCSlya5u+/n20nelmRZkp1JdrfHpW18knw4yWySe5Kc\n0/daG9v43Uk2Hs8NkyQ9swxzHUCSk4B9wHnAFuBgVV2T5EpgaVX9RpKLgF8HLmrjPlRV5yVZBuwC\nZoAC7gJeXVWHRrpFkqSBDHsIaC3wtap6CNgAbG/924GLW3sDcEP1fB44LcnpwAXAzqo62D70dwLr\nj3kLJElHZcmQ4y8FPtbaK6rq4dZ+BFjR2iuBPX3P2dv65ut/miSbgc0Ap5xyyqvPOuusIUuUpG67\n6667vlFVyxcaN3AAJDkZ+GXgXYevq6pKMpJ7SlTVVmArwMzMTO3atWsULytJnZHkoUHGDXMI6ELg\ni1X1aFt+tB3aoT3ub/37gNV9z1vV+ubrlyRNwDABcBk/PPwDsAOYm8mzEbi5r//yNhvofOCxdqjo\nVmBdkqVtxtC61idJmoCBDgElOQV4HfCrfd3XADcl2QQ8BFzS+m+hNwNoFngcuAKgqg4muRq4s417\nT1UdPOYtkCQdlaGmgY6b5wAkaXhJ7qqqmYXGeSWwJHWUASBJHWUASFJHGQCS1FHDXgm86J1x5acm\nXcJAHrzm9ZMuQdKUcw9AkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoA\nkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6aqAASHJakk8k+UqS+5P8TJJlSXYm\n2d0el7axSfLhJLNJ7klyTt/rbGzjdyfZeLw2SpK0sEH3AD4EfLqqzgJeAdwPXAncVlVrgNvaMsCF\nwJr2sxm4DiDJMuAq4DzgXOCqudCQJI3fggGQ5PnAzwPXA1TV96rqW8AGYHsbth24uLU3ADdUz+eB\n05KcDlwA7Kyqg1V1CNgJrB/p1kiSBjbIHsCZwAHgD5J8KclHkpwCrKiqh9uYR4AVrb0S2NP3/L2t\nb77+p0myOcmuJLsOHDgw3NZIkgY2SAAsAc4BrquqVwH/wA8P9wBQVQXUKAqqqq1VNVNVM8uXLx/F\nS0qSjmCQANgL7K2qL7TlT9ALhEfboR3a4/62fh+wuu/5q1rffP2SpAlYMACq6hFgT5KXtq61wH3A\nDmBuJs9G4ObW3gFc3mYDnQ881g4V3QqsS7K0nfxd1/okSROwZMBxvw58NMnJwAPAFfTC46Ykm4CH\ngEva2FuAi4BZ4PE2lqo6mORq4M427j1VdXAkWyFJGtpAAVBVdwMzR1i19ghjC9gyz+tsA7YNU6Ak\n6fjwSmBJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIA\nJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpqoABI8mCS\nLye5O8mu1rcsyc4ku9vj0tafJB9OMpvkniTn9L3OxjZ+d5KNx2eTJEmDGGYP4F9W1SuraqYtXwnc\nVlVrgNvaMsCFwJr2sxm4DnqBAVwFnAecC1w1FxqSpPE7lkNAG4Dtrb0duLiv/4bq+TxwWpLTgQuA\nnVV1sKoOATuB9cfw+yVJx2DQACjgL5LclWRz61tRVQ+39iPAitZeCezpe+7e1jdf/9Mk2ZxkV5Jd\nBw4cGLA8SdKwlgw47jVVtS/Ji4CdSb7Sv7KqKkmNoqCq2gpsBZiZmRnJa0qSftRAewBVta897gf+\njN4x/EfboR3a4/42fB+wuu/pq1rffP2SpAlYMACSnJLkeXNtYB3wN8AOYG4mz0bg5tbeAVzeZgOd\nDzzWDhXdCqxLsrSd/F3X+iRJEzDIIaAVwJ8lmRv/x1X16SR3Ajcl2QQ8BFzSxt8CXATMAo8DVwBU\n1cEkVwN3tnHvqaqDI9sSSdJQFgyAqnoAeMUR+r8JrD1CfwFb5nmtbcC24cuUJI2aVwJLUkcZAJLU\nUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLU\nUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRAwdAkpOSfCnJn7flM5N8Icls\nko8nObn1/1hbnm3rz+h7jXe1/q8muWDUGyNJGtwwewBvBe7vW/4d4NqqeglwCNjU+jcBh1r/tW0c\nSc4GLgVeBqwHfj/JScdWviTpaA0UAElWAa8HPtKWA7wW+EQbsh24uLU3tGXa+rVt/Abgxqp6oqr+\nFpgFzh3FRkiShjfoHsB/B/4L8P22/ALgW1X1ZFveC6xs7ZXAHoC2/rE2/gf9R3jODyTZnGRXkl0H\nDhwYYlMkScNYMACS/BKwv6ruGkM9VNXWqpqpqpnly5eP41dKUictGWDMzwK/nOQi4DnAqcCHgNOS\nLGnf8lcB+9r4fcBqYG+SJcDzgW/29c/pf44kacwW3AOoqndV1aqqOoPeSdzPVNWbgduBN7ZhG4Gb\nW3tHW6at/0xVVeu/tM0SOhNYA9wxsi2RJA1lkD2A+fwGcGOS3wa+BFzf+q8H/ijJLHCQXmhQVfcm\nuQm4D3gS2FJVTx3D75ckHYOhAqCqPgt8trUf4AizeKrqu8Cb5nn+e4H3DlukJGn0vBJYkjrKAJCk\njjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCk\njjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOWjAAkjwnyR1J/jrJvUne3frP\nTPKFJLNJPp7k5Nb/Y215tq0/o++13tX6v5rkguO1UZKkhQ2yB/AE8NqqegXwSmB9kvOB3wGuraqX\nAIeATW38JuBQ67+2jSPJ2cClwMuA9cDvJzlplBsjSRrcggFQPd9pi89uPwW8FvhE698OXNzaG9oy\nbf3aJGn9N1bVE1X1t8AscO5ItkKSNLSBzgEkOSnJ3cB+YCfwNeBbVfVkG7IXWNnaK4E9AG39Y8AL\n+vuP8Jz+37U5ya4kuw4cODD8FkmSBjJQAFTVU1X1SmAVvW/tZx2vgqpqa1XNVNXM8uXLj9evkaTO\nG2oWUFV9C7gd+BngtCRL2qpVwL7W3gesBmjrnw98s7//CM+RJI3ZILOAlic5rbV/HHgdcD+9IHhj\nG7YRuLm1d7Rl2vrPVFW1/kvbLKEzgTXAHaPaEEnScJYsPITTge1txs6zgJuq6s+T3AfcmOS3gS8B\n17fx1wN/lGQWOEhv5g9VdW+Sm4D7gCeBLVX11Gg3R5I0qAUDoKruAV51hP4HOMIsnqr6LvCmeV7r\nvcB7hy9TkjRqXgksSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJH\nGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJH\nLRgASVYnuT3JfUnuTfLW1r8syc4ku9vj0tafJB9OMpvkniTn9L3WxjZ+d5KNx2+zJEkLGWQP4Eng\nnVV1NnA+sCXJ2cCVwG1VtQa4rS0DXAisaT+bgeugFxjAVcB5wLnAVXOhIUkavwUDoKoerqovtvbf\nA/cDK4ENwPY2bDtwcWtvAG6ons8DpyU5HbgA2FlVB6vqELATWD/SrZEkDWyocwBJzgBeBXwBWFFV\nD7dVjwArWnslsKfvaXtb33z9h/+OzUl2Jdl14MCBYcqTJA1h4ABI8hPAnwBvq6pv96+rqgJqFAVV\n1daqmqmqmeXLl4/iJSVJRzBQACR5Nr0P/49W1Z+27kfboR3a4/7Wvw9Y3ff0Va1vvn5J0gQMMgso\nwPXA/VX1wb5VO4C5mTwbgZv7+i9vs4HOBx5rh4puBdYlWdpO/q5rfZKkCVgywJifBd4CfDnJ3a3v\nN4FrgJuSbAIeAi5p624BLgJmgceBKwCq6mCSq4E727j3VNXBkWyFJGloCwZAVf1fIPOsXnuE8QVs\nmee1tgHbhilQknR8eCWwJHWUASBJHWUASFJHGQCS1FGDzAKS5nXGlZ+adAkDefCa10+6BOmE4x6A\nJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaA\nJHWUASBJHWUASFJHGQCS1FEGgCR11IIBkGRbkv1J/qavb1mSnUl2t8elrT9JPpxkNsk9Sc7pe87G\nNn53ko3HZ3MkSYMaZA/gD4H1h/VdCdxWVWuA29oywIXAmvazGbgOeoEBXAWcB5wLXDUXGpKkyVgw\nAKrqc8DBw7o3ANtbeztwcV//DdXzeeC0JKcDFwA7q+pgVR0CdvKjoSJJGqOjPQewoqoebu1HgBWt\nvRLY0zdub+ubr/9HJNmcZFeSXQcOHDjK8iRJCznmk8BVVUCNoJa519taVTNVNbN8+fJRvawk6TBH\nGwCPtkM7tMf9rX8fsLpv3KrWN1+/JGlCjjYAdgBzM3k2Ajf39V/eZgOdDzzWDhXdCqxLsrSd/F3X\n+iRJE7JkoQFJPgb8IvDCJHvpzea5BrgpySbgIeCSNvwW4CJgFngcuAKgqg4muRq4s417T1UdfmJZ\nkjRGCwZAVV02z6q1RxhbwJZ5XmcbsG2o6iRJx82CASBpfM648lOTLmEgD17z+kmXoBEwACRNJcN0\nYd4LSJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ\n6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqqLEHQJL1Sb6aZDbJleP+/ZKk\nnrEGQJKTgP8JXAicDVyW5Oxx1iBJ6hn3HsC5wGxVPVBV3wNuBDaMuQZJEpCqGt8vS94IrK+qf9+W\n3wKcV1W/1jdmM7C5Lb4U+OrYCjx6LwS+Mekipojv52j5fo7OYnkvf7Kqli80aMk4KhlGVW0Ftk66\njmEk2VVVM5OuY1r4fo6W7+foTNt7Oe5DQPuA1X3Lq1qfJGnMxh0AdwJrkpyZ5GTgUmDHmGuQJDHm\nQ0BV9WSSXwNuBU4CtlXVveOs4ThZVIesFgHfz9Hy/RydqXovx3oSWJJ04vBKYEnqKANAkjrKAJCk\njjIAJKmjDACpI5L8yqRrWEySPDfJs/uWX5rk7UneMMm6RslZQENK8nvAfG/aE8DXgI9W1d+Pr6rF\nK8nLgJ+qqh1t+Vrg+W31/6iqL06suCmT5OtV9eJJ17FYJPkcsKmqdid5CXAH8FF6N7K8o6reNdEC\nR8AAGFKSjc+wegnwMuDlVfW6MZW0qCX538B/q6q/asv3Af8VeC7wK1V18STrmyZJ9lTV6oVHCiDJ\nl6vq5a19NbCsqra0i1jvmlu3mJ1w9wI60VXV9oXGJLllHLVMidPnPvybb1fVnwAk+dUJ1TSt/LY3\nnP7367XA+wGq6ntJvj+ZkkbLABhSkhcCW4BDwDZ6fxQ/R+/QzzuraraqLppgiYvN8/oXqur8vsUX\njbmWRS/JlznyB32AFWMuZ7G7J8nv0rtf2UuAvwBIctpEqxohA2B4fwzsAtbQOyb4B8CH6IXAR4Bf\nnFhli9PfJTmvqr7Q35nkfODvJlTTYvZLky5givwH4K3AGcC6qnq89Z8N/O6kiholzwEMKclfV9Ur\nkgR4qP+kWpK7q+qVEyxv0UlyLvBx4A+BuRO+rwY2Av+6qu6YUGlTJclrgMuqasuka1lskjyH3h4A\n9P6h1XcnWc8oOQ10eE8BVC85D//HEFNxXHCc2gf8efRuDvhv28+zgPP98D82SV6V5P1JHgSuBr4y\n4ZIWlSRLkrwP2ANsB24A9iR5X//00MXMPYAhJfkW8Dl6x1R/rrVpy6+pqqWTqm0xSnJqVX17nnUv\nrqqvj7umxSzJTwOXtZ9v0Nu7+s9V9ZMTLWwRalOSnwe8fW5ad5JT6R3++ceqeusk6xsFA2BISX6h\nNX+c3nmAAmaBfwSoqr+cUGmLUpIvVtU5rX1bVa090joNps1O+T/05q/Ptr4HquqfT7ayxSfJbuCn\n67APySQnAV+pqjWTqWx0PAk8vL8C3gv8O2Du2+lqesewf3NCNS1m6Wsve4Z1Gswb6P2jpduTfBq4\nEd/Ho1WHf/i3zqeSTMU3Z88BDO99wFLgzKo6p31D/Sl6V69OxcyAMat52kda1gKq6pNVdSlwFnA7\n8DbgRUmuS7JustUtOvclufzwziT/hik5n+IhoCF1YbdwnJLsBT5I71vq21ubtvw2r1w9dkmWAm+i\nN6tq7ULj1ZNkJfCn9A7v3tW6Z+gd/v1XVbXo/5+5h4CGN/W7hWP2v/jhxWD9behdV6FjVFWHktyE\nF9YNpX3An5fktfRu8QJwS1XdNsGyRsoAGN59SS6vqhv6O6dpt3Ccqurdk65hmiRZTe9eSv8M+CS9\nCxevBt4CfGyCpS06bf7/f6R3DcCXgeur6snJVjVaHgIaUhd2C8cpyW89w+qqqqvHVswUSHI78JfA\n/wPWA2uBe+lNZXxkkrUtNkk+DvwTvVlVFwIPVtXbJlvVaBkAR+mw3cL7pmm3cJySvPMI3acAm4AX\nVNVPjLmkRW3uSvW+5UeBF1fVExMsa1E67G6gS+jdAnqqpiV7COgoVdVngM9Muo7Frqo+MNdO8jx6\n9165gt70xQ/M9zzNr530nZv6+Qjw3CSnAFTVwYkVtvj801yjqp7s3f1lurgHoIlLsgx4B/Bmepfc\nf6iqDk22qsWp3fbh+xx57n95QdjgkjwF/MPcIr3DvI+3dlXVqZOqbVTcA9BEJXk/vYuXttL7Rzrf\nmXBJi90vVNVDky5iGlTVSZOu4XhzD0AT1W5d8ATwJE+/8GtqvmWNk7fP0DDcA9BEVZVXo4/W9B2o\n1nHjHoA0RZLsp3cC/Yiq6j+NsRyd4NwDkKZL//Up0jNyD0CaIp4D0DA8/ipNl+9NugAtHu4BSFMk\nyav50Vtsf6Oq9kyoJJ3ADABpirR7AR1uGXAyvX8Kf/eYS9IJzACQOiDJDPDBqvr5SdeiE4fnAKQO\nqKpdgDfW09MYAFIHJFmB/2JTh/E6AGmKJPk9fvSDfhnwL+jdaVX6Ac8BSFMkycbDugr4JnBnVe2f\nQEk6gRkA0hRJcmpVfXuedS+uqq+PuyaduDwHIE2Xz841khz+X+o+Od5SdKIzAKTp0n830GXPsE4y\nAKQpc/hVwPOtk5wFJE2ZFyV5B71v+3Nt2vLyyZWlE5EngaUpkuSqZ1pfVe8eVy068RkAktRRHgKS\npkiS33qG1VVVV4+tGJ3w3AOQpkiSdx6h+xRgE/CCqvJ+QPoBA0CaUkmeR+/2D5uAm4APeDWw+nkI\nSJoySZYB7wDeDGwHzqmqQ5OtSiciA0CaIkneD7wB2Aq8vKq+M+GSdALzEJA0RZJ8H3gCeJKnX/gV\neieBT51IYTohGQCS1FHeCkKSOsoAkKSOMgAkqaMMAEnqqP8PterCAdFVyOQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x145ed630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "s2 = df.Classification\n",
    "print(s2.value_counts())\n",
    "# s2.value_counts().plot(kind='hist') \n",
    "s2.value_counts().plot( kind=\"bar\") \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "# import matplotlib.pyplot as plt\n",
    "# import numpy as np\n",
    "\n",
    "# y = np.random.rand(10,4)\n",
    "# y[:,0]= np.arange(10)\n",
    "# df = pd.DataFrame(y, columns=[\"X\", \"A\", \"B\", \"C\"])\n",
    "\n",
    "# ax = df.plot(x=\"X\", y=\"A\", kind=\"bar\")\n",
    "# df.plot(x=\"X\", y=\"B\", kind=\"bar\", ax=ax, color=\"C2\")\n",
    "# df.plot(x=\"X\", y=\"C\", kind=\"bar\", ax=ax, color=\"C3\")\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step _: Data PreProcessing ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ara = AraTweet()\n",
    "\n",
    "(Data,rating)=ara.read_clean_reviews()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10006"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "le = preprocessing.LabelEncoder()\n",
    "\n",
    "target_rating = le.fit_transform(rating)\n",
    "# list(le.inverse_transform([0,1,2,3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10006L,)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_rating.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\x00'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_rating.data[10005]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([u'NEG', u'NEUTRAL', u'OBJ', u'POS'], dtype='<U7')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 3, 0, ..., 2, 3, 0], dtype=int64)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_rating"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### String Token Count(Bag of Words):\n",
    "\n",
    "It takes each sentence (all the words) present in the data set in the review section and then splits each of the words present in the form of tokens. The occurrence of these tokens in the whole data set are counted in such a way that the count of the occurrence of each token in a positive and negative feedback (in balanced dataset) or positive, negative and neutral (in unbalanced dataset) are collected separately. Finally, the word frequency of the tokens is calculated.\n",
    "\n",
    "code:  https://stackoverflow.com/questions/653887/equivalent-for-linkedhashmap-in-python\n",
    "\n",
    "Ref: https://acadpubl.eu/hub/2018-119-12/articles/5/1211.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10006, 37555)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "count_vect = CountVectorizer()\n",
    "X_train_counts = count_vect.fit_transform(Data)\n",
    "X_train_counts.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Above ‘count_vect.fit_transform(twenty_train.data)’, create words/tokens dictionary. such that [n_samples, n_features]."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Frequency–inverse document frequency(Tfidf):\n",
    "\n",
    "\n",
    "It measures how important a word is to differentiate each category. It reduce the weightage of more common words like (stop words or common words) which occurs in all tweets. \n",
    "\n",
    "Code: https://towardsdatascience.com/another-twitter-sentiment-analysis-with-python-part-5-50b4e87d9bdd \n",
    "\n",
    "Ref: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.121.1424&rep=rep1&type=pdf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10006, 37555)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)\n",
    "X_train_tfidf.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we will now split the data (both features and their labels) into training and test sets. 80% of the data will be used for training and 20% for testing similar to the benchmark model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set has 8004 samples.\n",
      "Testing set has 2002 samples.\n"
     ]
    }
   ],
   "source": [
    "# Import train_test_split\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "# Split the 'features' and 'income' data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train_tfidf, \n",
    "                                                    rating, \n",
    "                                                    test_size = 0.2, \n",
    "                                                    random_state = 0)\n",
    "\n",
    "# Show the results of the split\n",
    "print(\"Training set has {} samples.\".format(X_train.shape[0]))\n",
    "print(\"Testing set has {} samples.\".format(X_test.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OBJ        6691\n",
      "NEG        1684\n",
      "NEUTRAL     832\n",
      "POS         799\n",
      "Name: rating, dtype: int64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAYAAAC926RRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFpZJREFUeJzt3X+MXWd95/H3pzFpSwrEJoOVtZ0mW1xQEALCKEm39McS\n4fygqrMU2CCWeLPedVdyu/xabUOlbQQp2hQKLHS3kbzErVNRQkRb4i0RqRVC2VUXEgfS0CQgD2mC\n7SaxwSaUpoQmfPeP+wzcmJnMvfb1vZ573i9pNM95znPvfM+VNZ85z3nOcaoKSVL3/MikC5AkTYYB\nIEkdZQBIUkcZAJLUUQaAJHWUASBJHbVkACR5QZK7+r6+leQtSVYl2ZVkT/u+so1Pkg8lmUtyd5Jz\n+t5rUxu/J8mm43lgkqSnl2HuA0hyErAfOA/YChyqqmuSXAmsrKrfSHIJ8OvAJW3cB6vqvCSrgN3A\nLFDAncDLq+rwSI9IkjSQYaeALgC+WlUPAhuBHa1/B3Bpa28Erq+ezwGnJjkduBDYVVWH2i/9XcBF\nx3wEkqSjsmLI8ZcBH23t1VX1UGs/DKxu7TXA3r7X7Gt9i/Uv6rTTTqszzzxzyBIlqdvuvPPOr1fV\nzFLjBg6AJCcDvwy848h9VVVJRvJMiSRbgC0AZ5xxBrt37x7F20pSZyR5cJBxw0wBXQx8oaoeaduP\ntKkd2vcDrX8/sK7vdWtb32L9T1FV26pqtqpmZ2aWDDBJ0lEaJgDewA+mfwB2AvMreTYBN/X1X95W\nA50PPNqmim4BNiRZ2VYMbWh9kqQJGGgKKMkpwKuAX+3rvga4Mclm4EHg9a3/ZnorgOaAx4ArAKrq\nUJKrgTvauHdV1aFjPgJJ0lEZahnouM3OzpbXACRpOEnurKrZpcZ5J7AkdZQBIEkdZQBIUkcZAJLU\nUcPeCbzsnXnlJyddwkAeuObVky5B0pTzDECSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmj\nDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpqoABIcmqSjyf5\ncpL7kvxMklVJdiXZ076vbGOT5ENJ5pLcneScvvfZ1MbvSbLpeB2UJGlpg54BfBD4VFW9EHgJcB9w\nJXBrVa0Hbm3bABcD69vXFuBagCSrgKuA84BzgavmQ0OSNH5LBkCS5wA/D1wHUFXfrapvAhuBHW3Y\nDuDS1t4IXF89nwNOTXI6cCGwq6oOVdVhYBdw0UiPRpI0sEHOAM4CDgJ/kOSLST6c5BRgdVU91MY8\nDKxu7TXA3r7X72t9i/VLkiZgkABYAZwDXFtVLwP+gR9M9wBQVQXUKApKsiXJ7iS7Dx48OIq3lCQt\nYJAA2Afsq6rPt+2P0wuER9rUDu37gbZ/P7Cu7/VrW99i/U9RVduqaraqZmdmZoY5FknSEJYMgKp6\nGNib5AWt6wLgXmAnML+SZxNwU2vvBC5vq4HOBx5tU0W3ABuSrGwXfze0PknSBKwYcNyvAx9JcjJw\nP3AFvfC4Mclm4EHg9W3szcAlwBzwWBtLVR1KcjVwRxv3rqo6NJKjkCQNbaAAqKq7gNkFdl2wwNgC\nti7yPtuB7cMUKEk6PrwTWJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrK\nAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrK\nAJCkjhooAJI8kORLSe5Ksrv1rUqyK8me9n1l60+SDyWZS3J3knP63mdTG78nyabjc0iSpEEMcwbw\nL6vqpVU127avBG6tqvXArW0b4GJgffvaAlwLvcAArgLOA84FrpoPDUnS+B3LFNBGYEdr7wAu7eu/\nvno+B5ya5HTgQmBXVR2qqsPALuCiY/j5kqRjMGgAFPAXSe5MsqX1ra6qh1r7YWB1a68B9va9dl/r\nW6xfkjQBKwYc94qq2p/kecCuJF/u31lVlaRGUVALmC0AZ5xxxijeUpK0gIHOAKpqf/t+APgzenP4\nj7SpHdr3A234fmBd38vXtr7F+o/8WduqaraqZmdmZoY7GknSwJYMgCSnJHnWfBvYAPwNsBOYX8mz\nCbiptXcCl7fVQOcDj7apoluADUlWtou/G1qfJGkCBpkCWg38WZL58X9cVZ9KcgdwY5LNwIPA69v4\nm4FLgDngMeAKgKo6lORq4I427l1VdWhkRyJJGsqSAVBV9wMvWaD/G8AFC/QXsHWR99oObB++TEnS\nqHknsCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS\n1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHTVwACQ5KckX\nk/x52z4ryeeTzCX5WJKTW/+Ptu25tv/Mvvd4R+v/SpILR30wkqTBDXMG8Gbgvr7t3wE+UFXPBw4D\nm1v/ZuBw6/9AG0eSs4HLgBcBFwG/n+SkYytfknS0BgqAJGuBVwMfbtsBXgl8vA3ZAVza2hvbNm3/\nBW38RuCGqnq8qv4WmAPOHcVBSJKGN+gZwH8H/gvwvbb9XOCbVfVE294HrGntNcBegLb/0Tb++/0L\nvEaSNGZLBkCSXwIOVNWdY6iHJFuS7E6y++DBg+P4kZLUSYOcAfws8MtJHgBuoDf180Hg1CQr2pi1\nwP7W3g+sA2j7nwN8o79/gdd8X1Vtq6rZqpqdmZkZ+oAkSYNZMgCq6h1VtbaqzqR3EffTVfVG4Dbg\ntW3YJuCm1t7Ztmn7P11V1fova6uEzgLWA7eP7EgkSUNZsfSQRf0GcEOS3wa+CFzX+q8D/ijJHHCI\nXmhQVfckuRG4F3gC2FpVTx7Dz5ckHYOhAqCqPgN8prXvZ4FVPFX1HeB1i7z+3cC7hy1SkjR63gks\nSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEg\nSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHLRkASX4sye1J/jrJ\nPUne2frPSvL5JHNJPpbk5Nb/o217ru0/s++93tH6v5LkwuN1UJKkpQ1yBvA48MqqegnwUuCiJOcD\nvwN8oKqeDxwGNrfxm4HDrf8DbRxJzgYuA14EXAT8fpKTRnkwkqTBLRkA1fPttvmM9lXAK4GPt/4d\nwKWtvbFt0/ZfkCSt/4aqeryq/haYA84dyVFIkoY20DWAJCcluQs4AOwCvgp8s6qeaEP2AWtaew2w\nF6DtfxR4bn//Aq+RJI3ZQAFQVU9W1UuBtfT+an/h8SooyZYku5PsPnjw4PH6MZLUeUOtAqqqbwK3\nAT8DnJpkRdu1Ftjf2vuBdQBt/3OAb/T3L/Ca/p+xrapmq2p2ZmZmmPIkSUMYZBXQTJJTW/vHgVcB\n99ELgte2YZuAm1p7Z9um7f90VVXrv6ytEjoLWA/cPqoDkSQNZ8XSQzgd2NFW7PwIcGNV/XmSe4Eb\nkvw28EXgujb+OuCPkswBh+it/KGq7klyI3Av8ASwtaqeHO3hSJIGtWQAVNXdwMsW6L+fBVbxVNV3\ngNct8l7vBt49fJmSpFHzTmBJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnq\nKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnq\nKANAkjpqyQBIsi7JbUnuTXJPkje3/lVJdiXZ076vbP1J8qEkc0nuTnJO33ttauP3JNl0/A5LkrSU\nQc4AngDeXlVnA+cDW5OcDVwJ3FpV64Fb2zbAxcD69rUFuBZ6gQFcBZwHnAtcNR8akqTxWzIAquqh\nqvpCa/89cB+wBtgI7GjDdgCXtvZG4Prq+RxwapLTgQuBXVV1qKoOA7uAi0Z6NJKkgQ11DSDJmcDL\ngM8Dq6vqobbrYWB1a68B9va9bF/rW6xfkjQBAwdAkp8A/gR4S1V9q39fVRVQoygoyZYku5PsPnjw\n4CjeUpK0gIECIMkz6P3y/0hV/WnrfqRN7dC+H2j9+4F1fS9f2/oW63+KqtpWVbNVNTszMzPMsUiS\nhjDIKqAA1wH3VdX7+3btBOZX8mwCburrv7ytBjofeLRNFd0CbEiysl383dD6JEkTsGKAMT8LvAn4\nUpK7Wt9vAtcANybZDDwIvL7tuxm4BJgDHgOuAKiqQ0muBu5o495VVYdGchSSpKEtGQBV9X+BLLL7\nggXGF7B1kffaDmwfpkBJ0vHhncCS1FEGgCR1lAEgSR1lAEhSRw2yCkha1JlXfnLSJQzkgWtePekS\npBOOZwCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUA\nSFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRSwZAku1JDiT5m76+VUl2JdnTvq9s/UnyoSRzSe5O\nck7faza18XuSbDo+hyNJGtQgZwB/CFx0RN+VwK1VtR64tW0DXAysb19bgGuhFxjAVcB5wLnAVfOh\nIUmajCUDoKo+Cxw6onsjsKO1dwCX9vVfXz2fA05NcjpwIbCrqg5V1WFgFz8cKpKkMTraawCrq+qh\n1n4YWN3aa4C9feP2tb7F+n9Iki1JdifZffDgwaMsT5K0lGO+CFxVBdQIapl/v21VNVtVszMzM6N6\nW0nSEY42AB5pUzu07wda/35gXd+4ta1vsX5J0oQcbQDsBOZX8mwCburrv7ytBjofeLRNFd0CbEiy\nsl383dD6JEkTsmKpAUk+CvwicFqSffRW81wD3JhkM/Ag8Po2/GbgEmAOeAy4AqCqDiW5GrijjXtX\nVR15YVmSNEZLBkBVvWGRXRcsMLaArYu8z3Zg+1DVSZKOmyUDQNL4nHnlJyddwkAeuObVky5BI2AA\nSJpKhunSfBaQJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBI\nUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRYw+AJBcl+UqSuSRX\njvvnS5J6xhoASU4C/idwMXA28IYkZ4+zBklSz7jPAM4F5qrq/qr6LnADsHHMNUiSgFTV+H5Y8lrg\noqr69237TcB5VfVrfWO2AFva5guAr4ytwKN3GvD1SRcxRfw8R8vPc3SWy2f5k1U1s9SgFeOoZBhV\ntQ3YNuk6hpFkd1XNTrqOaeHnOVp+nqMzbZ/luKeA9gPr+rbXtj5J0piNOwDuANYnOSvJycBlwM4x\n1yBJYsxTQFX1RJJfA24BTgK2V9U946zhOFlWU1bLgJ/naPl5js5UfZZjvQgsSTpxeCewJHWUASBJ\nHWUASFJHGQCS1FEGgNQRSX5l0jUsJ0memeQZfdsvSPLWJK+ZZF2j5CqgISX5PWCxD+1x4KvAR6rq\n78dX1fKV5EXAT1XVzrb9AeA5bff/qKovTKy4KZPka1V1xqTrWC6SfBbYXFV7kjwfuB34CL0HWd5e\nVe+YaIEjYAAMKcmmp9m9AngR8OKqetWYSlrWkvxv4L9V1V+17XuB/wo8E/iVqrp0kvVNkyR7q2rd\n0iMFkORLVfXi1r4aWFVVW9tNrHfO71vOTrhnAZ3oqmrHUmOS3DyOWqbE6fO//JtvVdWfACT51QnV\nNK38a284/Z/XK4H3AlTVd5N8bzIljZYBMKQkpwFbgcPAdnr/KH6O3tTP26tqrqoumWCJy82z+jeq\n6vy+zeeNuZZlL8mXWPgXfYDVYy5nubs7ye/Se17Z84G/AEhy6kSrGiEDYHh/DOwG1tObE/wD4IP0\nQuDDwC9OrLLl6e+SnFdVn+/vTHI+8HcTqmk5+6VJFzBF/gPwZuBMYENVPdb6zwZ+d1JFjZLXAIaU\n5K+r6iVJAjzYf1EtyV1V9dIJlrfsJDkX+Bjwh8D8Bd+XA5uAf11Vt0+otKmS5BXAG6pq66RrWW6S\n/Bi9MwDo/YdW35lkPaPkMtDhPQlQveQ88j+GmIp5wXFqv+DPo/dwwH/bvn4EON9f/scmycuSvDfJ\nA8DVwJcnXNKykmRFkvcAe4EdwPXA3iTv6V8eupx5BjCkJN8EPktvTvXnWpu2/YqqWjmp2pajJM+u\nqm8tsu+MqvrauGtazpL8NPCG9vV1emdX/7mqfnKihS1DbUnys4C3zi/rTvJsetM//1hVb55kfaNg\nAAwpyS+05o/Tuw5QwBzwjwBV9ZcTKm1ZSvKFqjqntW+tqgsW2qfBtNUp/4fe+vW51nd/Vf3zyVa2\n/CTZA/x0HfFLMslJwJerav1kKhsdLwIP76+AdwP/Dpj/63QdvTns35xQTctZ+tqrnmafBvMaev/R\n0m1JPgXcgJ/j0aojf/m3zieTTMVfzl4DGN57gJXAWVV1TvsL9afo3b06FSsDxqwWaS+0rSVU1Seq\n6jLghcBtwFuA5yW5NsmGyVa37Nyb5PIjO5P8G6bkeopTQEPqwmnhOCXZB7yf3l+pb21t2vZbvHP1\n2CVZCbyO3qqqC5Yar54ka4A/pTe9e2frnqU3/fuvqmrZ/3/mTgENb+pPC8fsf/GDm8H629C7r0LH\nqKoOJ7kRb6wbSvsFf16SV9J7xAvAzVV16wTLGikDYHj3Jrm8qq7v75ym08Jxqqp3TrqGaZJkHb1n\nKf0z4BP0bly8GngT8NEJlrbstPX//5HePQBfAq6rqicmW9VoOQU0pC6cFo5Tkt96mt1VVVePrZgp\nkOQ24C+B/wdcBFwA3ENvKePDk6xtuUnyMeCf6K2quhh4oKreMtmqRssAOEpHnBbeO02nheOU5O0L\ndJ8CbAaeW1U/MeaSlrX5O9X7th8BzqiqxydY1rJ0xNNAV9B7BPRULUt2CugoVdWngU9Puo7lrqre\nN99O8ix6z165gt7yxfct9jotrl30nV/6+TDwzCSnAFTVoYkVtvz803yjqp7oPf1lungGoIlLsgp4\nG/BGerfcf7CqDk+2quWpPfbheyy89r+8IWxwSZ4E/mF+k94072OtXVX17EnVNiqeAWiikryX3s1L\n2+j9RzrfnnBJy90vVNWDky5iGlTVSZOu4XjzDEAT1R5d8DjwBE+98Wtq/soaJx+foWF4BqCJqirv\nRh+t6Zuo1nHjGYA0RZIcoHcBfUFV9Z/GWI5OcJ4BSNOl//4U6Wl5BiBNEa8BaBjOv0rT5buTLkDL\nh2cA0hRJ8nJ++BHbX6+qvRMqSScwA0CaIu1ZQEdaBZxM7z+Fv2vMJekEZgBIHZBkFnh/Vf38pGvR\nicNrAFIHVNVuwAfr6SkMAKkDkqzG/2JTR/A+AGmKJPk9fvgX/SrgX9B70qr0fV4DkKZIkk1HdBXw\nDeCOqjowgZJ0AjMApCmS5NlV9a1F9p1RVV8bd006cXkNQJoun5lvJDnyf6n7xHhL0YnOAJCmS//T\nQFc9zT7JAJCmzJF3AS+2T3IVkDRlnpfkbfT+2p9v07ZnJleWTkReBJamSJKrnm5/Vb1zXLXoxGcA\nSFJHOQUkTZEkv/U0u6uqrh5bMTrheQYgTZEkb1+g+xRgM/DcqvJ5QPo+A0CaUkmeRe/xD5uBG4H3\neTew+jkFJE2ZJKuAtwFvBHYA51TV4clWpRORASBNkSTvBV4DbANeXFXfnnBJOoE5BSRNkSTfAx4H\nnuCpN36F3kXgZ0+kMJ2QDABJ6igfBSFJHWUASFJHGQCS1FEGgCR11P8HngfE7gVH2LgAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x201926d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "s2 = pd.DataFrame(rating,columns=['rating'])\n",
    "print(s2.rating.value_counts())\n",
    "# s2.value_counts().plot(kind='hist') \n",
    "s2.rating.value_counts().plot( kind=\"bar\") \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "clf = MultinomialNB().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predicted = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Accuracy score: ', '0.679320679321')\n",
      "('Precision score: ', '0.41991991992')\n",
      "('Recall score: ', '0.252732240437')\n",
      "('F1 score: ', '0.207729600876')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:1074: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:1074: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy score: ', format(accuracy_score(y_test, predicted)))\n",
    "print('Precision score: ', format(precision_score(y_test, predicted,average='macro')))\n",
    "print('Recall score: ', format(recall_score(y_test, predicted,average='macro')))\n",
    "print('F1 score: ', format(f1_score(y_test, predicted,average='macro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6724137931034483\n",
      "{'alpha': 0.3, 'fit_prior': True}\n"
     ]
    }
   ],
   "source": [
    "parameters = {'alpha': [1.0, .1 , .3 , .6 , .8 , 2.0], 'fit_prior': [True,False]}\n",
    "\n",
    "# X_train = feat_generator['feat_generator'].fit_transform(d_train)\n",
    "# X_test = feat_generator['feat_generator'].transform(d_test)\n",
    "\n",
    "gs_clf = GridSearchCV(clf, {'alpha': [1.0, .1 , .3 , .6 , .8 , 0.0], 'fit_prior': [True,False]}, n_jobs=10)\n",
    "gs_clf = gs_clf.fit(X_train, y_train)\n",
    "\n",
    "print(gs_clf.best_score_)\n",
    "print(gs_clf.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step _: Machine implementation using scikit-learn ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\ipykernel\\__main__.py:12: DeprecationWarning: Passing additional arguments to the metric function as **kwargs is deprecated and will no longer be supported in 0.18. Use metric_params instead.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>fold_idx</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>0</td>\n",
       "      <td>0.671992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>1</td>\n",
       "      <td>0.676324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>2</td>\n",
       "      <td>0.676662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>3</td>\n",
       "      <td>0.674663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>4</td>\n",
       "      <td>0.662331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           model_name  fold_idx  accuracy\n",
       "0  LogisticRegression         0  0.671992\n",
       "1  LogisticRegression         1  0.676324\n",
       "2  LogisticRegression         2  0.676662\n",
       "3  LogisticRegression         3  0.674663\n",
       "4  LogisticRegression         4  0.662331"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "models = [\n",
    "    LogisticRegression(random_state=0),\n",
    "    PassiveAggressiveClassifier(random_state=0),\n",
    "    LinearSVC(tol=1e-3, random_state=0),\n",
    "    Perceptron(n_iter=100, random_state=0),\n",
    "    BernoulliNB(binarize=0.5),\n",
    "    SGDClassifier(loss=\"hinge\", penalty=\"l2\", random_state=0),\n",
    "    KNeighborsClassifier(n_neighbors=5, metric='euclidean', random_state=0),\n",
    "    RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0),\n",
    "    DecisionTreeClassifier(),\n",
    "    MultinomialNB()\n",
    "]\n",
    "CV = 5\n",
    "cv_df = pd.DataFrame(index=range(CV * len(models)))\n",
    "entries = []\n",
    "for model in models:\n",
    "    model_name = model.__class__.__name__\n",
    "    accuracies = cross_val_score(model, X_train_tfidf, rating, scoring='accuracy', cv=CV)\n",
    "    for fold_idx, accuracy in enumerate(accuracies):\n",
    "        entries.append((model_name, fold_idx, accuracy))\n",
    "        \n",
    "        \n",
    "cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'])\n",
    "cv_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>fold_idx</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PassiveAggressiveClassifier</td>\n",
       "      <td>0</td>\n",
       "      <td>0.686970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PassiveAggressiveClassifier</td>\n",
       "      <td>2</td>\n",
       "      <td>0.680160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>LinearSVC</td>\n",
       "      <td>0</td>\n",
       "      <td>0.702446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LinearSVC</td>\n",
       "      <td>1</td>\n",
       "      <td>0.687313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>LinearSVC</td>\n",
       "      <td>2</td>\n",
       "      <td>0.697151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>SGDClassifier</td>\n",
       "      <td>0</td>\n",
       "      <td>0.696955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>SGDClassifier</td>\n",
       "      <td>1</td>\n",
       "      <td>0.691808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>SGDClassifier</td>\n",
       "      <td>2</td>\n",
       "      <td>0.698651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>SGDClassifier</td>\n",
       "      <td>3</td>\n",
       "      <td>0.682659</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model_name  fold_idx  accuracy\n",
       "5   PassiveAggressiveClassifier         0  0.686970\n",
       "7   PassiveAggressiveClassifier         2  0.680160\n",
       "10                    LinearSVC         0  0.702446\n",
       "11                    LinearSVC         1  0.687313\n",
       "12                    LinearSVC         2  0.697151\n",
       "25                SGDClassifier         0  0.696955\n",
       "26                SGDClassifier         1  0.691808\n",
       "27                SGDClassifier         2  0.698651\n",
       "28                SGDClassifier         3  0.682659"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_df[cv_df['accuracy'] > .68]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# TODO: Import two metrics from sklearn - fbeta_score and accuracy_score\n",
    "\n",
    "\n",
    "def train_predict(learner, sample_size, X_train, y_train, X_test, y_test): \n",
    "    '''\n",
    "    inputs:\n",
    "       - learner: the learning algorithm to be trained and predicted on\n",
    "       - sample_size: the size of samples (number) to be drawn from training set\n",
    "       - X_train: features training set\n",
    "       - y_train: income training set\n",
    "       - X_test: features testing set\n",
    "       - y_test: income testing set\n",
    "    '''\n",
    "    \n",
    "    results = {}\n",
    "    \n",
    "    # TODO: Fit the learner to the training data using slicing with 'sample_size' using .fit(training_features[:], training_labels[:])\n",
    "    start = time() # Get start time\n",
    "    \n",
    "    learner.fit(X_train[:sample_size] , y_train[:sample_size])\n",
    "    \n",
    "    end = time() # Get end time\n",
    "    \n",
    "    # TODO: Calculate the training time\n",
    "    results['train_time'] = end -start\n",
    "        \n",
    "    # TODO: Get the predictions on the test set(X_test),\n",
    "    #       then get predictions on the first 300 training samples(X_train) using .predict()\n",
    "    \n",
    "    start = time() # Get start time\n",
    "    \n",
    "    predictions_test = learner.predict(X_test)\n",
    "    predictions_train = learner.predict(X_train[:300])\n",
    "    \n",
    "    end = time() # Get end time\n",
    "    \n",
    "    # TODO: Calculate the total prediction time\n",
    "    results['pred_time'] = end - start\n",
    "            \n",
    "    # TODO: Compute accuracy on the first 300 training samples which is y_train[:300]\n",
    "    results['acc_train'] = accuracy_score(y_train[:300],predictions_train)\n",
    "        \n",
    "    # TODO: Compute accuracy on test set using accuracy_score()\n",
    "    results['acc_test'] = accuracy_score(y_test,predictions_test)\n",
    "    \n",
    "    # TODO: Compute F-score on the the first 300 training samples using fbeta_score()\n",
    "    results['f_train'] = f1_score(y_train[:300],predictions_train,average='binary',pos_label='POS')\n",
    "        \n",
    "    # TODO: Compute F-score on the test set which is y_test\n",
    "    results['f_test'] = f1_score(y_test,predictions_test,average='binary',pos_label='POS')\n",
    "       \n",
    "    # Success\n",
    "    print(\"{} trained on {} samples.\".format(learner.__class__.__name__, sample_size))\n",
    "        \n",
    "    # Return the results\n",
    "    return results , predictions_test , y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ref: Fixed using https://stackoverflow.com/questions/31421413/how-to-compute-precision-recall-accuracy-and-f1-score-for-the-multiclass-case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#this function to retrive sentimental count in each class\n",
    "def GetCount(array):\n",
    "    df = pd.DataFrame(array,columns= [\"Class\"])\n",
    "    counts = df.Class.value_counts()\n",
    "    OBJ = counts['OBJ']\n",
    "    NEG = counts['NEG']\n",
    "    NEUTRAL = counts['NEUTRAL']\n",
    "    POS = counts['POS']\n",
    "    return [OBJ, NEG, NEUTRAL ,POS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#this function to retrive sentimental count in each class\n",
    "def GetCount(array):\n",
    "    df = pd.DataFrame(array,columns= [\"Class\"])\n",
    "    counts = df.Class.value_counts()\n",
    "    OBJ = counts['OBJ']\n",
    "    NEG = counts['NEG']\n",
    "    NEUTRAL = counts['NEUTRAL']\n",
    "    POS = counts['POS']\n",
    "    return [OBJ, NEG, NEUTRAL ,POS]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## using differnt samples with models to train the classifiers without parameter tunining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "========================================\n",
      "('Loading data:', '4-balanced')\n",
      "\n",
      "========================================\n",
      "Logistic Regression\n",
      "LogisticRegression trained on 2560 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 2560 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 2560 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 2560 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 2560 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 2560 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 2560 samples.\n",
      "RandomForest\n",
      "RandomForestClassifier trained on 2560 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 2560 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 2560 samples.\n",
      "Logistic Regression\n",
      "LogisticRegression trained on 2560 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 2560 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 2560 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 2560 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 2560 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 2560 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 2560 samples.\n",
      "RandomForest\n",
      "RandomForestClassifier trained on 2560 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 2560 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 2560 samples.\n",
      "Logistic Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression trained on 2560 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 2560 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 2560 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 2560 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 2560 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 2560 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 2560 samples.\n",
      "RandomForest\n",
      "RandomForestClassifier trained on 2560 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 2560 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 2560 samples.\n",
      "Logistic Regression\n",
      "LogisticRegression trained on 2560 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 2560 samples.\n",
      "SVM\n",
      "LinearSVC trained on 2560 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 2560 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 2560 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 2560 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 2560 samples.\n",
      "RandomForest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier trained on 2560 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 2560 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 2560 samples.\n",
      "Logistic Regression\n",
      "LogisticRegression trained on 2560 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 2560 samples.\n",
      "SVM\n",
      "LinearSVC trained on 2560 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 2560 samples.\n",
      "bnb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BernoulliNB trained on 2560 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 2560 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 2560 samples.\n",
      "RandomForest\n",
      "RandomForestClassifier trained on 2560 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 2560 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 2560 samples.\n",
      "Logistic Regression\n",
      "LogisticRegression trained on 2560 samples.\n",
      "Passive Aggresive\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PassiveAggressiveClassifier trained on 2560 samples.\n",
      "SVM\n",
      "LinearSVC trained on 2560 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 2560 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 2560 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 2560 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 2560 samples.\n",
      "RandomForest\n",
      "RandomForestClassifier trained on 2560 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 2560 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 2560 samples.\n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  69.0   82.0   90.0      NEG   0.456954  0.433962  Perceptron   \n",
      "1  57.0   91.0  102.0  NEUTRAL   0.385135  0.358491  Perceptron   \n",
      "2  82.0  101.0   77.0      OBJ   0.448087  0.515723  Perceptron   \n",
      "3  73.0   81.0   86.0      POS   0.474026  0.459119  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.441824  0.440621  \n",
      "1         tfidf_ng3  0.441824  0.440621  \n",
      "2         tfidf_ng3  0.441824  0.440621  \n",
      "3         tfidf_ng3  0.441824  0.440621  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP    FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  71.0  95.0   88.0      NEG   0.427711  0.446541  Perceptron   \n",
      "1  52.0  83.0  107.0  NEUTRAL   0.385185  0.327044  Perceptron   \n",
      "2  77.0  82.0   82.0      OBJ   0.484277  0.484277  Perceptron   \n",
      "3  85.0  91.0   74.0      POS   0.482955  0.534591  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.448113  0.445601  \n",
      "1         tfidf_ng2  0.448113  0.445601  \n",
      "2         tfidf_ng2  0.448113  0.445601  \n",
      "3         tfidf_ng2  0.448113  0.445601  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  73.0   96.0   86.0      NEG   0.431953  0.459119  Perceptron   \n",
      "1  46.0   91.0  113.0  NEUTRAL   0.335766  0.289308  Perceptron   \n",
      "2  90.0  114.0   69.0      OBJ   0.441176  0.566038  Perceptron   \n",
      "3  64.0   62.0   95.0      POS   0.507937  0.402516  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.429245  0.425231  \n",
      "1         tfidf_ng1  0.429245  0.425231  \n",
      "2         tfidf_ng1  0.429245  0.425231  \n",
      "3         tfidf_ng1  0.429245  0.425231  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  63.0   89.0   96.0      NEG   0.414474  0.396226        SVM   \n",
      "1  54.0   70.0  105.0  NEUTRAL   0.435484  0.339623        SVM   \n",
      "2  75.0   77.0   84.0      OBJ   0.493421  0.471698        SVM   \n",
      "3  93.0  115.0   66.0      POS   0.447115  0.584906        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.448113  0.443974  \n",
      "1         count_ng2  0.448113  0.443974  \n",
      "2         count_ng2  0.448113  0.443974  \n",
      "3         count_ng2  0.448113  0.443974  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  68.0   72.0  91.0      NEG   0.485714  0.427673        mnb   \n",
      "1  77.0  131.0  82.0  NEUTRAL   0.370192  0.484277        mnb   \n",
      "2  89.0   73.0  70.0      OBJ   0.549383  0.559748        mnb   \n",
      "3  76.0   50.0  83.0      POS   0.603175  0.477987        mnb   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng2  0.487421  0.49058  \n",
      "1         count_ng2  0.487421  0.49058  \n",
      "2         count_ng2  0.487421  0.49058  \n",
      "3         count_ng2  0.487421  0.49058  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   78.0  94.0      NEG   0.454545  0.408805        mnb   \n",
      "1  73.0  131.0  86.0  NEUTRAL   0.357843  0.459119        mnb   \n",
      "2  87.0   78.0  72.0      OBJ   0.527273  0.547170        mnb   \n",
      "3  73.0   51.0  86.0      POS   0.588710  0.459119        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.468553  0.471401  \n",
      "1         count_ng1  0.468553  0.471401  \n",
      "2         count_ng1  0.468553  0.471401  \n",
      "3         count_ng1  0.468553  0.471401  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   89.0   94.0      NEG   0.422078  0.408805        SVM   \n",
      "1  47.0   77.0  112.0  NEUTRAL   0.379032  0.295597        SVM   \n",
      "2  79.0   94.0   80.0      OBJ   0.456647  0.496855        SVM   \n",
      "3  82.0  103.0   77.0      POS   0.443243  0.515723        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.429245  0.425035  \n",
      "1         count_ng1  0.429245  0.425035  \n",
      "2         count_ng1  0.429245  0.425035  \n",
      "3         count_ng1  0.429245  0.425035  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  56.0   65.0  103.0      NEG   0.462810  0.352201  Logistic Regression   \n",
      "1  74.0  114.0   85.0  NEUTRAL   0.393617  0.465409  Logistic Regression   \n",
      "2  89.0  113.0   70.0      OBJ   0.440594  0.559748  Logistic Regression   \n",
      "3  65.0   60.0   94.0      POS   0.520000  0.408805  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.446541  0.444334  \n",
      "1         tfidf_ng3  0.446541  0.444334  \n",
      "2         tfidf_ng3  0.446541  0.444334  \n",
      "3         tfidf_ng3  0.446541  0.444334  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  58.0   78.0  101.0      NEG   0.426471  0.364780  Logistic Regression   \n",
      "1  71.0  110.0   88.0  NEUTRAL   0.392265  0.446541  Logistic Regression   \n",
      "2  85.0   98.0   74.0      OBJ   0.464481  0.534591  Logistic Regression   \n",
      "3  73.0   63.0   86.0      POS   0.536765  0.459119  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.451258  0.450715  \n",
      "1         tfidf_ng2  0.451258  0.450715  \n",
      "2         tfidf_ng2  0.451258  0.450715  \n",
      "3         tfidf_ng2  0.451258  0.450715  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP    FP    FN        L  Percision    Recall           Classifier  \\\n",
      "0  65.0  92.0  94.0      NEG   0.414013  0.408805  Logistic Regression   \n",
      "1  64.0  97.0  95.0  NEUTRAL   0.397516  0.402516  Logistic Regression   \n",
      "2  85.0  92.0  74.0      OBJ   0.480226  0.534591  Logistic Regression   \n",
      "3  74.0  67.0  85.0      POS   0.524823  0.465409  Logistic Regression   \n",
      "\n",
      "  feauter_generator      acc       f1  \n",
      "0         tfidf_ng1  0.45283  0.45267  \n",
      "1         tfidf_ng1  0.45283  0.45267  \n",
      "2         tfidf_ng1  0.45283  0.45267  \n",
      "3         tfidf_ng1  0.45283  0.45267  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  53.0   73.0  106.0      NEG   0.420635  0.333333  Perceptron   \n",
      "1  65.0  115.0   94.0  NEUTRAL   0.361111  0.408805  Perceptron   \n",
      "2  60.0   56.0   99.0      OBJ   0.517241  0.377358  Perceptron   \n",
      "3  95.0  119.0   64.0      POS   0.443925  0.597484  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.429245  0.425289  \n",
      "1         count_ng3  0.429245  0.425289  \n",
      "2         count_ng3  0.429245  0.425289  \n",
      "3         count_ng3  0.429245  0.425289  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  61.0   73.0   98.0      NEG   0.455224  0.383648  Perceptron   \n",
      "1  56.0   88.0  103.0  NEUTRAL   0.388889  0.352201  Perceptron   \n",
      "2  67.0   73.0   92.0      OBJ   0.478571  0.421384  Perceptron   \n",
      "3  94.0  124.0   65.0      POS   0.431193  0.591195  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.437107  0.433213  \n",
      "1         count_ng2  0.437107  0.433213  \n",
      "2         count_ng2  0.437107  0.433213  \n",
      "3         count_ng2  0.437107  0.433213  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  60.0   56.0   99.0      NEG   0.517241  0.377358  Perceptron   \n",
      "1  76.0  145.0   83.0  NEUTRAL   0.343891  0.477987  Perceptron   \n",
      "2  84.0  110.0   75.0      OBJ   0.432990  0.528302  Perceptron   \n",
      "3  53.0   52.0  106.0      POS   0.504762  0.333333  Perceptron   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng1  0.429245  0.42845  \n",
      "1         count_ng1  0.429245  0.42845  \n",
      "2         count_ng1  0.429245  0.42845  \n",
      "3         count_ng1  0.429245  0.42845  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  63.0  101.0   96.0      NEG   0.384146  0.396226        KNN   \n",
      "1  59.0  119.0  100.0  NEUTRAL   0.331461  0.371069        KNN   \n",
      "2  80.0   87.0   79.0      OBJ   0.479042  0.503145        KNN   \n",
      "3  58.0   69.0  101.0      POS   0.456693  0.364780        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.408805  0.409158  \n",
      "1         tfidf_ng2  0.408805  0.409158  \n",
      "2         tfidf_ng2  0.408805  0.409158  \n",
      "3         tfidf_ng2  0.408805  0.409158  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  58.0   76.0  101.0      NEG   0.432836  0.364780        sgd   \n",
      "1  73.0  112.0   86.0  NEUTRAL   0.394595  0.459119        sgd   \n",
      "2  87.0   81.0   72.0      OBJ   0.517857  0.547170        sgd   \n",
      "3  80.0   69.0   79.0      POS   0.536913  0.503145        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.468553  0.467978  \n",
      "1         tfidf_ng2  0.468553  0.467978  \n",
      "2         tfidf_ng2  0.468553  0.467978  \n",
      "3         tfidf_ng2  0.468553  0.467978  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   89.0   94.0      NEG   0.422078  0.408805        sgd   \n",
      "1  55.0  100.0  104.0  NEUTRAL   0.354839  0.345912        sgd   \n",
      "2  87.0  103.0   72.0      OBJ   0.457895  0.547170        sgd   \n",
      "3  69.0   68.0   90.0      POS   0.503650  0.433962        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.433962  0.432609  \n",
      "1         tfidf_ng1  0.433962  0.432609  \n",
      "2         tfidf_ng1  0.433962  0.432609  \n",
      "3         tfidf_ng1  0.433962  0.432609  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  69.0   73.0  90.0      NEG   0.485915  0.433962        mnb   \n",
      "1  80.0  131.0  79.0  NEUTRAL   0.379147  0.503145        mnb   \n",
      "2  86.0   72.0  73.0      OBJ   0.544304  0.540881        mnb   \n",
      "3  75.0   50.0  84.0      POS   0.600000  0.471698        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.487421  0.490415  \n",
      "1         count_ng3  0.487421  0.490415  \n",
      "2         count_ng3  0.487421  0.490415  \n",
      "3         count_ng3  0.487421  0.490415  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  62.0   84.0  97.0      NEG   0.424658  0.389937        sgd   \n",
      "1  73.0  117.0  86.0  NEUTRAL   0.384211  0.459119        sgd   \n",
      "2  83.0   81.0  76.0      OBJ   0.506098  0.522013        sgd   \n",
      "3  72.0   64.0  87.0      POS   0.529412  0.452830        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.455975  0.456741  \n",
      "1         tfidf_ng3  0.455975  0.456741  \n",
      "2         tfidf_ng3  0.455975  0.456741  \n",
      "3         tfidf_ng3  0.455975  0.456741  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  89.0  225.0   70.0      NEG   0.283439  0.559748        KNN   \n",
      "1   3.0    4.0  156.0  NEUTRAL   0.428571  0.018868        KNN   \n",
      "2  50.0  132.0  109.0      OBJ   0.274725  0.314465        KNN   \n",
      "3  33.0  100.0  126.0      POS   0.248120  0.207547        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.275157  0.232937  \n",
      "1         count_ng3  0.275157  0.232937  \n",
      "2         count_ng3  0.275157  0.232937  \n",
      "3         count_ng3  0.275157  0.232937  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  87.0  109.0   72.0      NEG   0.443878  0.547170        sgd   \n",
      "1  41.0   80.0  118.0  NEUTRAL   0.338843  0.257862        sgd   \n",
      "2  61.0   82.0   98.0      OBJ   0.426573  0.383648        sgd   \n",
      "3  80.0   96.0   79.0      POS   0.454545  0.503145        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.422956  0.416146  \n",
      "1         count_ng1  0.422956  0.416146  \n",
      "2         count_ng1  0.422956  0.416146  \n",
      "3         count_ng1  0.422956  0.416146  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  56.0   92.0  103.0      NEG   0.378378  0.352201        sgd   \n",
      "1  54.0  100.0  105.0  NEUTRAL   0.350649  0.339623        sgd   \n",
      "2  75.0   71.0   84.0      OBJ   0.513699  0.471698        sgd   \n",
      "3  92.0   96.0   67.0      POS   0.489362  0.578616        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.435535  0.432983  \n",
      "1         count_ng2  0.435535  0.432983  \n",
      "2         count_ng2  0.435535  0.432983  \n",
      "3         count_ng2  0.435535  0.432983  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   86.0  94.0      NEG   0.430464  0.408805        sgd   \n",
      "1  64.0  108.0  95.0  NEUTRAL   0.372093  0.402516        sgd   \n",
      "2  73.0   75.0  86.0      OBJ   0.493243  0.459119        sgd   \n",
      "3  85.0   80.0  74.0      POS   0.515152  0.534591        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.451258  0.451581  \n",
      "1         count_ng3  0.451258  0.451581  \n",
      "2         count_ng3  0.451258  0.451581  \n",
      "3         count_ng3  0.451258  0.451581  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   82.0  99.0      NEG   0.422535  0.377358        SVM   \n",
      "1  71.0  113.0  88.0  NEUTRAL   0.385870  0.446541        SVM   \n",
      "2  89.0   83.0  70.0      OBJ   0.517442  0.559748        SVM   \n",
      "3  75.0   63.0  84.0      POS   0.543478  0.471698        SVM   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         tfidf_ng2  0.463836  0.46387  \n",
      "1         tfidf_ng2  0.463836  0.46387  \n",
      "2         tfidf_ng2  0.463836  0.46387  \n",
      "3         tfidf_ng2  0.463836  0.46387  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  71.0   90.0   88.0      NEG   0.440994  0.446541  Passive Aggresive   \n",
      "1  57.0   90.0  102.0  NEUTRAL   0.387755  0.358491  Passive Aggresive   \n",
      "2  86.0  102.0   73.0      OBJ   0.457447  0.540881  Passive Aggresive   \n",
      "3  70.0   70.0   89.0      POS   0.500000  0.440252  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.446541  0.445051  \n",
      "1         tfidf_ng1  0.446541  0.445051  \n",
      "2         tfidf_ng1  0.446541  0.445051  \n",
      "3         tfidf_ng1  0.446541  0.445051  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall         Classifier  \\\n",
      "0  64.0   85.0  95.0      NEG   0.429530  0.402516  Passive Aggresive   \n",
      "1  67.0  109.0  92.0  NEUTRAL   0.380682  0.421384  Passive Aggresive   \n",
      "2  90.0   83.0  69.0      OBJ   0.520231  0.566038  Passive Aggresive   \n",
      "3  74.0   64.0  85.0      POS   0.536232  0.465409  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.463836  0.464017  \n",
      "1         tfidf_ng2  0.463836  0.464017  \n",
      "2         tfidf_ng2  0.463836  0.464017  \n",
      "3         tfidf_ng2  0.463836  0.464017  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall         Classifier  \\\n",
      "0  60.0   82.0  99.0      NEG   0.422535  0.377358  Passive Aggresive   \n",
      "1  71.0  112.0  88.0  NEUTRAL   0.387978  0.446541  Passive Aggresive   \n",
      "2  85.0   91.0  74.0      OBJ   0.482955  0.534591  Passive Aggresive   \n",
      "3  70.0   65.0  89.0      POS   0.518519  0.440252  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.449686  0.449382  \n",
      "1         tfidf_ng3  0.449686  0.449382  \n",
      "2         tfidf_ng3  0.449686  0.449382  \n",
      "3         tfidf_ng3  0.449686  0.449382  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  51.0  102.0  108.0      NEG   0.333333  0.320755  DecisionTree   \n",
      "1  51.0   87.0  108.0  NEUTRAL   0.369565  0.320755  DecisionTree   \n",
      "2  59.0  108.0  100.0      OBJ   0.353293  0.371069  DecisionTree   \n",
      "3  62.0  116.0   97.0      POS   0.348315  0.389937  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.350629  0.350068  \n",
      "1         tfidf_ng1  0.350629  0.350068  \n",
      "2         tfidf_ng1  0.350629  0.350068  \n",
      "3         tfidf_ng1  0.350629  0.350068  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  43.0   68.0  116.0      NEG   0.387387  0.270440  DecisionTree   \n",
      "1  50.0   88.0  109.0  NEUTRAL   0.362319  0.314465  DecisionTree   \n",
      "2  81.0  124.0   78.0      OBJ   0.395122  0.509434  DecisionTree   \n",
      "3  68.0  114.0   91.0      POS   0.373626  0.427673  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.380503  0.374775  \n",
      "1         tfidf_ng3  0.380503  0.374775  \n",
      "2         tfidf_ng3  0.380503  0.374775  \n",
      "3         tfidf_ng3  0.380503  0.374775  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  40.0   78.0  119.0      NEG   0.338983  0.251572  DecisionTree   \n",
      "1  44.0   93.0  115.0  NEUTRAL   0.321168  0.276730  DecisionTree   \n",
      "2  69.0  118.0   90.0      OBJ   0.368984  0.433962  DecisionTree   \n",
      "3  71.0  123.0   88.0      POS   0.365979  0.446541  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.352201  0.346804  \n",
      "1         tfidf_ng2  0.352201  0.346804  \n",
      "2         tfidf_ng2  0.352201  0.346804  \n",
      "3         tfidf_ng2  0.352201  0.346804  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  49.0   82.0  110.0      NEG   0.374046  0.308176  RandomForest   \n",
      "1  55.0  103.0  104.0  NEUTRAL   0.348101  0.345912  RandomForest   \n",
      "2  68.0   76.0   91.0      OBJ   0.472222  0.427673  RandomForest   \n",
      "3  95.0  108.0   64.0      POS   0.467980  0.597484  RandomForest   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng1  0.419811  0.41466  \n",
      "1         count_ng1  0.419811  0.41466  \n",
      "2         count_ng1  0.419811  0.41466  \n",
      "3         count_ng1  0.419811  0.41466  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  50.0   77.0  109.0      NEG   0.393701  0.314465  RandomForest   \n",
      "1  56.0  101.0  103.0  NEUTRAL   0.356688  0.352201  RandomForest   \n",
      "2  56.0   63.0  103.0      OBJ   0.470588  0.352201  RandomForest   \n",
      "3  94.0  139.0   65.0      POS   0.403433  0.591195  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.402516  0.396638  \n",
      "1         count_ng3  0.402516  0.396638  \n",
      "2         count_ng3  0.402516  0.396638  \n",
      "3         count_ng3  0.402516  0.396638  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  56.0   88.0  103.0      NEG   0.388889  0.352201  RandomForest   \n",
      "1  58.0   92.0  101.0  NEUTRAL   0.386667  0.364780  RandomForest   \n",
      "2  58.0   54.0  101.0      OBJ   0.517857  0.364780  RandomForest   \n",
      "3  99.0  131.0   60.0      POS   0.430435  0.622642  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.426101  0.420521  \n",
      "1         count_ng2  0.426101  0.420521  \n",
      "2         count_ng2  0.426101  0.420521  \n",
      "3         count_ng2  0.426101  0.420521  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   93.0   99.0      NEG   0.392157  0.377358        SVM   \n",
      "1  51.0   67.0  108.0  NEUTRAL   0.432203  0.320755        SVM   \n",
      "2  71.0   79.0   88.0      OBJ   0.473333  0.446541        SVM   \n",
      "3  94.0  121.0   65.0      POS   0.437209  0.591195        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.433962  0.428767  \n",
      "1         count_ng3  0.433962  0.428767  \n",
      "2         count_ng3  0.433962  0.428767  \n",
      "3         count_ng3  0.433962  0.428767  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  59.0   92.0  100.0      NEG   0.390728  0.371069  Logistic Regression   \n",
      "1  53.0   74.0  106.0  NEUTRAL   0.417323  0.333333  Logistic Regression   \n",
      "2  71.0   74.0   88.0      OBJ   0.489655  0.446541  Logistic Regression   \n",
      "3  96.0  117.0   63.0      POS   0.450704  0.603774  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.438679  0.433627  \n",
      "1         count_ng3  0.438679  0.433627  \n",
      "2         count_ng3  0.438679  0.433627  \n",
      "3         count_ng3  0.438679  0.433627  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  67.0   84.0  92.0      NEG   0.443709  0.421384        SVM   \n",
      "1  64.0  105.0  95.0  NEUTRAL   0.378698  0.402516        SVM   \n",
      "2  86.0   88.0  73.0      OBJ   0.494253  0.540881        SVM   \n",
      "3  75.0   67.0  84.0      POS   0.528169  0.471698        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.459119  0.459339  \n",
      "1         tfidf_ng1  0.459119  0.459339  \n",
      "2         tfidf_ng1  0.459119  0.459339  \n",
      "3         tfidf_ng1  0.459119  0.459339  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  62.0  175.0   97.0      NEG   0.261603  0.389937        KNN   \n",
      "1   6.0   20.0  153.0  NEUTRAL   0.230769  0.037736        KNN   \n",
      "2  59.0  124.0  100.0      OBJ   0.322404  0.371069        KNN   \n",
      "3  56.0  134.0  103.0      POS   0.294737  0.352201        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.287736  0.260986  \n",
      "1         count_ng1  0.287736  0.260986  \n",
      "2         count_ng1  0.287736  0.260986  \n",
      "3         count_ng1  0.287736  0.260986  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-count_ng1\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   66.0   75.0   93.0      NEG   0.468085  0.415094        bnb   \n",
      "1   32.0   37.0  127.0  NEUTRAL   0.463768  0.201258        bnb   \n",
      "2   90.0   99.0   69.0      OBJ   0.476190  0.566038        bnb   \n",
      "3  109.0  128.0   50.0      POS   0.459916  0.685535        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.466981  0.447112  \n",
      "1         count_ng1  0.466981  0.447112  \n",
      "2         count_ng1  0.466981  0.447112  \n",
      "3         count_ng1  0.466981  0.447112  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-count_ng2\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   48.0   46.0  111.0      NEG   0.510638  0.301887        bnb   \n",
      "1   16.0    6.0  143.0  NEUTRAL   0.727273  0.100629        bnb   \n",
      "2   86.0   91.0   73.0      OBJ   0.485876  0.540881        bnb   \n",
      "3  124.0  219.0   35.0      POS   0.361516  0.779874        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.430818  0.390543  \n",
      "1         count_ng2  0.430818  0.390543  \n",
      "2         count_ng2  0.430818  0.390543  \n",
      "3         count_ng2  0.430818  0.390543  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  62.0   97.0   97.0      NEG   0.389937  0.389937  Logistic Regression   \n",
      "1  53.0   74.0  106.0  NEUTRAL   0.417323  0.333333  Logistic Regression   \n",
      "2  72.0   73.0   87.0      OBJ   0.496552  0.452830  Logistic Regression   \n",
      "3  96.0  109.0   63.0      POS   0.468293  0.603774  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.444969  0.440431  \n",
      "1         count_ng2  0.444969  0.440431  \n",
      "2         count_ng2  0.444969  0.440431  \n",
      "3         count_ng2  0.444969  0.440431  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  65.0   92.0   94.0      NEG   0.414013  0.408805  Logistic Regression   \n",
      "1  52.0   76.0  107.0  NEUTRAL   0.406250  0.327044  Logistic Regression   \n",
      "2  80.0   79.0   79.0      OBJ   0.503145  0.503145  Logistic Regression   \n",
      "3  91.0  101.0   68.0      POS   0.473958  0.572327  Logistic Regression   \n",
      "\n",
      "  feauter_generator      acc        f1  \n",
      "0         count_ng1  0.45283  0.448856  \n",
      "1         count_ng1  0.45283  0.448856  \n",
      "2         count_ng1  0.45283  0.448856  \n",
      "3         count_ng1  0.45283  0.448856  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  64.0   92.0   95.0      NEG   0.410256  0.402516  Passive Aggresive   \n",
      "1  51.0   65.0  108.0  NEUTRAL   0.439655  0.320755  Passive Aggresive   \n",
      "2  81.0   98.0   78.0      OBJ   0.452514  0.509434  Passive Aggresive   \n",
      "3  83.0  102.0   76.0      POS   0.448649  0.522013  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.438679  0.434777  \n",
      "1         count_ng2  0.438679  0.434777  \n",
      "2         count_ng2  0.438679  0.434777  \n",
      "3         count_ng2  0.438679  0.434777  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  65.0   90.0   94.0      NEG   0.419355  0.408805  Passive Aggresive   \n",
      "1  52.0   61.0  107.0  NEUTRAL   0.460177  0.327044  Passive Aggresive   \n",
      "2  82.0   99.0   77.0      OBJ   0.453039  0.515723  Passive Aggresive   \n",
      "3  85.0  102.0   74.0      POS   0.454545  0.534591  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.446541  0.442512  \n",
      "1         count_ng3  0.446541  0.442512  \n",
      "2         count_ng3  0.446541  0.442512  \n",
      "3         count_ng3  0.446541  0.442512  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  61.0  107.0   98.0      NEG   0.363095  0.383648        KNN   \n",
      "1  48.0  123.0  111.0  NEUTRAL   0.280702  0.301887        KNN   \n",
      "2  77.0  103.0   82.0      OBJ   0.427778  0.484277        KNN   \n",
      "3  49.0   68.0  110.0      POS   0.418803  0.308176        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.369497  0.368337  \n",
      "1         tfidf_ng1  0.369497  0.368337  \n",
      "2         tfidf_ng1  0.369497  0.368337  \n",
      "3         tfidf_ng1  0.369497  0.368337  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-count_ng3\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   36.0   23.0  123.0      NEG   0.610169  0.226415        bnb   \n",
      "1    6.0    5.0  153.0  NEUTRAL   0.545455  0.037736        bnb   \n",
      "2   70.0   69.0   89.0      OBJ   0.503597  0.440252        bnb   \n",
      "3  139.0  288.0   20.0      POS   0.325527  0.874214        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.394654  0.336266  \n",
      "1         count_ng3  0.394654  0.336266  \n",
      "2         count_ng3  0.394654  0.336266  \n",
      "3         count_ng3  0.394654  0.336266  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  76.0  212.0   83.0      NEG   0.263889  0.477987        KNN   \n",
      "1   5.0   12.0  154.0  NEUTRAL   0.294118  0.031447        KNN   \n",
      "2  53.0  102.0  106.0      OBJ   0.341935  0.333333        KNN   \n",
      "3  44.0  132.0  115.0      POS   0.250000  0.276730        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.279874  0.249282  \n",
      "1         count_ng2  0.279874  0.249282  \n",
      "2         count_ng2  0.279874  0.249282  \n",
      "3         count_ng2  0.279874  0.249282  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   68.0  99.0      NEG   0.468750  0.377358        mnb   \n",
      "1  82.0  132.0  77.0  NEUTRAL   0.383178  0.515723        mnb   \n",
      "2  89.0   72.0  70.0      OBJ   0.552795  0.559748        mnb   \n",
      "3  77.0   56.0  82.0      POS   0.578947  0.484277        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.484277  0.485361  \n",
      "1         tfidf_ng3  0.484277  0.485361  \n",
      "2         tfidf_ng3  0.484277  0.485361  \n",
      "3         tfidf_ng3  0.484277  0.485361  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   69.0  99.0      NEG   0.465116  0.377358        mnb   \n",
      "1  83.0  130.0  76.0  NEUTRAL   0.389671  0.522013        mnb   \n",
      "2  90.0   72.0  69.0      OBJ   0.555556  0.566038        mnb   \n",
      "3  79.0   53.0  80.0      POS   0.598485  0.496855        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.490566  0.491652  \n",
      "1         tfidf_ng2  0.490566  0.491652  \n",
      "2         tfidf_ng2  0.490566  0.491652  \n",
      "3         tfidf_ng2  0.490566  0.491652  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  67.0   80.0  92.0      NEG   0.455782  0.421384        mnb   \n",
      "1  79.0  128.0  80.0  NEUTRAL   0.381643  0.496855        mnb   \n",
      "2  86.0   68.0  73.0      OBJ   0.558442  0.540881        mnb   \n",
      "3  75.0   53.0  84.0      POS   0.585938  0.471698        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.482704  0.485443  \n",
      "1         tfidf_ng1  0.482704  0.485443  \n",
      "2         tfidf_ng1  0.482704  0.485443  \n",
      "3         tfidf_ng1  0.482704  0.485443  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  40.0   85.0  119.0      NEG   0.320000  0.251572  RandomForest   \n",
      "1  57.0  110.0  102.0  NEUTRAL   0.341317  0.358491  RandomForest   \n",
      "2  66.0   70.0   93.0      OBJ   0.485294  0.415094  RandomForest   \n",
      "3  88.0  120.0   71.0      POS   0.423077  0.553459  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.394654  0.389601  \n",
      "1         tfidf_ng1  0.394654  0.389601  \n",
      "2         tfidf_ng1  0.394654  0.389601  \n",
      "3         tfidf_ng1  0.394654  0.389601  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall    Classifier  \\\n",
      "0  62.0  103.0  97.0      NEG   0.375758  0.389937  RandomForest   \n",
      "1  68.0  105.0  91.0  NEUTRAL   0.393064  0.427673  RandomForest   \n",
      "2  60.0   73.0  99.0      OBJ   0.451128  0.377358  RandomForest   \n",
      "3  76.0   89.0  83.0      POS   0.460606  0.477987  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.418239  0.418112  \n",
      "1         tfidf_ng3  0.418239  0.418112  \n",
      "2         tfidf_ng3  0.418239  0.418112  \n",
      "3         tfidf_ng3  0.418239  0.418112  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  45.0   83.0  114.0      NEG   0.351562  0.283019  RandomForest   \n",
      "1  64.0   92.0   95.0  NEUTRAL   0.410256  0.402516  RandomForest   \n",
      "2  73.0   90.0   86.0      OBJ   0.447853  0.459119  RandomForest   \n",
      "3  77.0  112.0   82.0      POS   0.407407  0.484277  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.407233  0.403971  \n",
      "1         tfidf_ng2  0.407233  0.403971  \n",
      "2         tfidf_ng2  0.407233  0.403971  \n",
      "3         tfidf_ng2  0.407233  0.403971  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  54.0  106.0  105.0      NEG   0.337500  0.339623  DecisionTree   \n",
      "1  48.0   99.0  111.0  NEUTRAL   0.326531  0.301887  DecisionTree   \n",
      "2  69.0  105.0   90.0      OBJ   0.396552  0.433962  DecisionTree   \n",
      "3  65.0   90.0   94.0      POS   0.419355  0.408805  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.371069  0.370178  \n",
      "1         count_ng1  0.371069  0.370178  \n",
      "2         count_ng1  0.371069  0.370178  \n",
      "3         count_ng1  0.371069  0.370178  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  53.0   75.0  106.0      NEG   0.414062  0.333333        SVM   \n",
      "1  74.0  123.0   85.0  NEUTRAL   0.375635  0.465409        SVM   \n",
      "2  88.0   91.0   71.0      OBJ   0.491620  0.553459        SVM   \n",
      "3  72.0   60.0   87.0      POS   0.545455  0.452830        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.451258  0.450156  \n",
      "1         tfidf_ng3  0.451258  0.450156  \n",
      "2         tfidf_ng3  0.451258  0.450156  \n",
      "3         tfidf_ng3  0.451258  0.450156  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  58.0  122.0  101.0      NEG   0.322222  0.364780  DecisionTree   \n",
      "1  48.0   94.0  111.0  NEUTRAL   0.338028  0.301887  DecisionTree   \n",
      "2  62.0  100.0   97.0      OBJ   0.382716  0.389937  DecisionTree   \n",
      "3  62.0   90.0   97.0      POS   0.407895  0.389937  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.361635  0.361532  \n",
      "1         count_ng3  0.361635  0.361532  \n",
      "2         count_ng3  0.361635  0.361532  \n",
      "3         count_ng3  0.361635  0.361532  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  54.0  111.0  105.0      NEG   0.327273  0.339623  DecisionTree   \n",
      "1  50.0   94.0  109.0  NEUTRAL   0.347222  0.314465  DecisionTree   \n",
      "2  66.0  120.0   93.0      OBJ   0.354839  0.415094  DecisionTree   \n",
      "3  60.0   81.0   99.0      POS   0.425532  0.377358  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.361635  0.361494  \n",
      "1         count_ng2  0.361635  0.361494  \n",
      "2         count_ng2  0.361635  0.361494  \n",
      "3         count_ng2  0.361635  0.361494  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-tfidf_ng1\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   66.0   75.0   93.0      NEG   0.468085  0.415094        bnb   \n",
      "1   32.0   37.0  127.0  NEUTRAL   0.463768  0.201258        bnb   \n",
      "2   90.0   99.0   69.0      OBJ   0.476190  0.566038        bnb   \n",
      "3  109.0  128.0   50.0      POS   0.459916  0.685535        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.466981  0.447112  \n",
      "1         tfidf_ng1  0.466981  0.447112  \n",
      "2         tfidf_ng1  0.466981  0.447112  \n",
      "3         tfidf_ng1  0.466981  0.447112  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-tfidf_ng2\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   48.0   46.0  111.0      NEG   0.510638  0.301887        bnb   \n",
      "1   16.0    6.0  143.0  NEUTRAL   0.727273  0.100629        bnb   \n",
      "2   86.0   91.0   73.0      OBJ   0.485876  0.540881        bnb   \n",
      "3  124.0  219.0   35.0      POS   0.361516  0.779874        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.430818  0.390543  \n",
      "1         tfidf_ng2  0.430818  0.390543  \n",
      "2         tfidf_ng2  0.430818  0.390543  \n",
      "3         tfidf_ng2  0.430818  0.390543  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-tfidf_ng3\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   36.0   23.0  123.0      NEG   0.610169  0.226415        bnb   \n",
      "1    6.0    5.0  153.0  NEUTRAL   0.545455  0.037736        bnb   \n",
      "2   70.0   69.0   89.0      OBJ   0.503597  0.440252        bnb   \n",
      "3  139.0  288.0   20.0      POS   0.325527  0.874214        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.394654  0.336266  \n",
      "1         tfidf_ng3  0.394654  0.336266  \n",
      "2         tfidf_ng3  0.394654  0.336266  \n",
      "3         tfidf_ng3  0.394654  0.336266  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  63.0  101.0   96.0      NEG   0.384146  0.396226        KNN   \n",
      "1  58.0  116.0  101.0  NEUTRAL   0.333333  0.364780        KNN   \n",
      "2  82.0   85.0   77.0      OBJ   0.491018  0.515723        KNN   \n",
      "3  58.0   73.0  101.0      POS   0.442748  0.364780        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.410377  0.410377  \n",
      "1         tfidf_ng3  0.410377  0.410377  \n",
      "2         tfidf_ng3  0.410377  0.410377  \n",
      "3         tfidf_ng3  0.410377  0.410377  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  65.0   83.0   94.0      NEG   0.439189  0.408805  Passive Aggresive   \n",
      "1  49.0   82.0  110.0  NEUTRAL   0.374046  0.308176  Passive Aggresive   \n",
      "2  88.0  119.0   71.0      OBJ   0.425121  0.553459  Passive Aggresive   \n",
      "3  72.0   78.0   87.0      POS   0.480000  0.452830  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.430818  0.427069  \n",
      "1         count_ng1  0.430818  0.427069  \n",
      "2         count_ng1  0.430818  0.427069  \n",
      "3         count_ng1  0.430818  0.427069  \n",
      "\n",
      "========================================\n",
      "('Loading data:', '4-unbalanced')\n",
      "\n",
      "========================================\n",
      "Logistic Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression trained on 8007 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 8007 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 8007 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 8007 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 8007 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 8007 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 8007 samples.\n",
      "RandomForest\n",
      "RandomForestClassifier trained on 8007 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 8007 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 8007 samples.\n",
      "Logistic Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression trained on 8007 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 8007 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 8007 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 8007 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 8007 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 8007 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 8007 samples.\n",
      "RandomForest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier trained on 8007 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 8007 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 8007 samples.\n",
      "Logistic Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression trained on 8007 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 8007 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 8007 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 8007 samples.\n",
      "bnb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BernoulliNB trained on 8007 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 8007 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 8007 samples.\n",
      "RandomForest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier trained on 8007 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 8007 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 8007 samples.\n",
      "Logistic Regression\n",
      "LogisticRegression trained on 8007 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 8007 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 8007 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 8007 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 8007 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 8007 samples.\n",
      "KNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNeighborsClassifier trained on 8007 samples.\n",
      "RandomForest\n",
      "RandomForestClassifier trained on 8007 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 8007 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 8007 samples.\n",
      "Logistic Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression trained on 8007 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 8007 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 8007 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 8007 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 8007 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 8007 samples.\n",
      "KNN\n",
      "KNeighborsClassifier trained on 8007 samples.\n",
      "RandomForest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier trained on 8007 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 8007 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 8007 samples.\n",
      "Logistic Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression trained on 8007 samples.\n",
      "Passive Aggresive\n",
      "PassiveAggressiveClassifier trained on 8007 samples.\n",
      "SVM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC trained on 8007 samples.\n",
      "Perceptron\n",
      "Perceptron trained on 8007 samples.\n",
      "bnb\n",
      "BernoulliNB trained on 8007 samples.\n",
      "sgd\n",
      "SGDClassifier trained on 8007 samples.\n",
      "KNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNeighborsClassifier trained on 8007 samples.\n",
      "RandomForest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier trained on 8007 samples.\n",
      "DecisionTree\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:976: DeprecationWarning: From version 0.18, binary input will not be handled specially when using averaged precision/recall/F-score. Please use average='binary' to report only the positive class performance.\n",
      "  'positive class performance.', DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier trained on 8007 samples.\n",
      "mnb\n",
      "MultinomialNB trained on 8007 samples.\n",
      "\n",
      "========================================\n",
      "4-unbalanced-SVM-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   100.0  135.0  236.0      NEG   0.425532  0.297619        SVM   \n",
      "1    11.0   49.0  155.0  NEUTRAL   0.183333  0.066265        SVM   \n",
      "2  1202.0  446.0  136.0      OBJ   0.729369  0.898356        SVM   \n",
      "3    29.0   27.0  130.0      POS   0.517857  0.182390        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.671336  0.627289  \n",
      "1         tfidf_ng1  0.671336  0.627289  \n",
      "2         tfidf_ng1  0.671336  0.627289  \n",
      "3         tfidf_ng1  0.671336  0.627289  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Logistic Regression-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0    38.0   36.0  298.0      NEG   0.513514  0.113095  Logistic Regression   \n",
      "1     2.0    3.0  164.0  NEUTRAL   0.400000  0.012048  Logistic Regression   \n",
      "2  1317.0  601.0   21.0      OBJ   0.686653  0.984305  Logistic Regression   \n",
      "3     2.0    0.0  157.0      POS   1.000000  0.012579  Logistic Regression   \n",
      "\n",
      "  feauter_generator      acc        f1  \n",
      "0         tfidf_ng1  0.67984  0.576546  \n",
      "1         tfidf_ng1  0.67984  0.576546  \n",
      "2         tfidf_ng1  0.67984  0.576546  \n",
      "3         tfidf_ng1  0.67984  0.576546  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Logistic Regression-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0    21.0   21.0  315.0      NEG   0.500000  0.062500  Logistic Regression   \n",
      "1     1.0    2.0  165.0  NEUTRAL   0.333333  0.006024  Logistic Regression   \n",
      "2  1328.0  624.0   10.0      OBJ   0.680328  0.992526  Logistic Regression   \n",
      "3     2.0    0.0  157.0      POS   1.000000  0.012579  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.676338  0.561985  \n",
      "1         tfidf_ng2  0.676338  0.561985  \n",
      "2         tfidf_ng2  0.676338  0.561985  \n",
      "3         tfidf_ng2  0.676338  0.561985  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-SVM-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   101.0  112.0  235.0      NEG   0.474178  0.300595        SVM   \n",
      "1    12.0   35.0  154.0  NEUTRAL   0.255319  0.072289        SVM   \n",
      "2  1238.0  458.0  100.0      OBJ   0.729953  0.925262        SVM   \n",
      "3    23.0   20.0  136.0      POS   0.534884  0.144654        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.687344  0.635549  \n",
      "1         tfidf_ng2  0.687344  0.635549  \n",
      "2         tfidf_ng2  0.687344  0.635549  \n",
      "3         tfidf_ng2  0.687344  0.635549  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  69.0   73.0  90.0      NEG   0.485915  0.433962        mnb   \n",
      "1  80.0  131.0  79.0  NEUTRAL   0.379147  0.503145        mnb   \n",
      "2  86.0   72.0  73.0      OBJ   0.544304  0.540881        mnb   \n",
      "3  75.0   50.0  84.0      POS   0.600000  0.471698        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.487421  0.490415  \n",
      "1         count_ng3  0.487421  0.490415  \n",
      "2         count_ng3  0.487421  0.490415  \n",
      "3         count_ng3  0.487421  0.490415  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  68.0   72.0  91.0      NEG   0.485714  0.427673        mnb   \n",
      "1  77.0  131.0  82.0  NEUTRAL   0.370192  0.484277        mnb   \n",
      "2  89.0   73.0  70.0      OBJ   0.549383  0.559748        mnb   \n",
      "3  76.0   50.0  83.0      POS   0.603175  0.477987        mnb   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng2  0.487421  0.49058  \n",
      "1         count_ng2  0.487421  0.49058  \n",
      "2         count_ng2  0.487421  0.49058  \n",
      "3         count_ng2  0.487421  0.49058  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   78.0  94.0      NEG   0.454545  0.408805        mnb   \n",
      "1  73.0  131.0  86.0  NEUTRAL   0.357843  0.459119        mnb   \n",
      "2  87.0   78.0  72.0      OBJ   0.527273  0.547170        mnb   \n",
      "3  73.0   51.0  86.0      POS   0.588710  0.459119        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.468553  0.471401  \n",
      "1         count_ng1  0.468553  0.471401  \n",
      "2         count_ng1  0.468553  0.471401  \n",
      "3         count_ng1  0.468553  0.471401  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  56.0   65.0  103.0      NEG   0.462810  0.352201  Logistic Regression   \n",
      "1  74.0  114.0   85.0  NEUTRAL   0.393617  0.465409  Logistic Regression   \n",
      "2  89.0  113.0   70.0      OBJ   0.440594  0.559748  Logistic Regression   \n",
      "3  65.0   60.0   94.0      POS   0.520000  0.408805  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.446541  0.444334  \n",
      "1         tfidf_ng3  0.446541  0.444334  \n",
      "2         tfidf_ng3  0.446541  0.444334  \n",
      "3         tfidf_ng3  0.446541  0.444334  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  58.0   78.0  101.0      NEG   0.426471  0.364780  Logistic Regression   \n",
      "1  71.0  110.0   88.0  NEUTRAL   0.392265  0.446541  Logistic Regression   \n",
      "2  85.0   98.0   74.0      OBJ   0.464481  0.534591  Logistic Regression   \n",
      "3  73.0   63.0   86.0      POS   0.536765  0.459119  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.451258  0.450715  \n",
      "1         tfidf_ng2  0.451258  0.450715  \n",
      "2         tfidf_ng2  0.451258  0.450715  \n",
      "3         tfidf_ng2  0.451258  0.450715  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP    FP    FN        L  Percision    Recall           Classifier  \\\n",
      "0  65.0  92.0  94.0      NEG   0.414013  0.408805  Logistic Regression   \n",
      "1  64.0  97.0  95.0  NEUTRAL   0.397516  0.402516  Logistic Regression   \n",
      "2  85.0  92.0  74.0      OBJ   0.480226  0.534591  Logistic Regression   \n",
      "3  74.0  67.0  85.0      POS   0.524823  0.465409  Logistic Regression   \n",
      "\n",
      "  feauter_generator      acc       f1  \n",
      "0         tfidf_ng1  0.45283  0.45267  \n",
      "1         tfidf_ng1  0.45283  0.45267  \n",
      "2         tfidf_ng1  0.45283  0.45267  \n",
      "3         tfidf_ng1  0.45283  0.45267  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-mnb-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision  Recall Classifier  \\\n",
      "0     0.0    1.0  336.0      NEG    0.00000     1.0        mnb   \n",
      "1     0.0    0.0  166.0  NEUTRAL    0.00000     0.0        mnb   \n",
      "2  1338.0  660.0    0.0      OBJ    0.66967     1.0        mnb   \n",
      "3     0.0    0.0  159.0      POS    0.00000     0.0        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.669335  0.536912  \n",
      "1         tfidf_ng1  0.669335  0.536912  \n",
      "2         tfidf_ng1  0.669335  0.536912  \n",
      "3         tfidf_ng1  0.669335  0.536912  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-bnb-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision  Recall Classifier  \\\n",
      "0     0.0    1.0  336.0      NEG    0.00000     1.0        bnb   \n",
      "1     0.0    0.0  166.0  NEUTRAL    0.00000     0.0        bnb   \n",
      "2  1338.0  660.0    0.0      OBJ    0.66967     1.0        bnb   \n",
      "3     0.0    0.0  159.0      POS    0.00000     0.0        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.669335  0.536912  \n",
      "1         count_ng3  0.669335  0.536912  \n",
      "2         count_ng3  0.669335  0.536912  \n",
      "3         count_ng3  0.669335  0.536912  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-bnb-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision  Recall Classifier  \\\n",
      "0     0.0    1.0  336.0      NEG    0.00000     1.0        bnb   \n",
      "1     0.0    0.0  166.0  NEUTRAL    0.00000     0.0        bnb   \n",
      "2  1338.0  660.0    0.0      OBJ    0.66967     1.0        bnb   \n",
      "3     0.0    0.0  159.0      POS    0.00000     0.0        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.669335  0.536912  \n",
      "1         count_ng2  0.669335  0.536912  \n",
      "2         count_ng2  0.669335  0.536912  \n",
      "3         count_ng2  0.669335  0.536912  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-bnb-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0     3.0    4.0  333.0      NEG   0.428571  0.008929        bnb   \n",
      "1     0.0    0.0  166.0  NEUTRAL   0.000000  0.000000        bnb   \n",
      "2  1336.0  656.0    2.0      OBJ   0.670683  0.998505        bnb   \n",
      "3     0.0    0.0  159.0      POS   0.000000  0.000000        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.669835  0.540016  \n",
      "1         count_ng1  0.669835  0.540016  \n",
      "2         count_ng1  0.669835  0.540016  \n",
      "3         count_ng1  0.669835  0.540016  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Logistic Regression-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0    15.0   17.0  321.0      NEG   0.468750  0.044643  Logistic Regression   \n",
      "1     1.0    1.0  165.0  NEUTRAL   0.500000  0.006024  Logistic Regression   \n",
      "2  1332.0  631.0    6.0      OBJ   0.678553  0.995516  Logistic Regression   \n",
      "3     2.0    0.0  157.0      POS   1.000000  0.012579  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.675338  0.556839  \n",
      "1         tfidf_ng3  0.675338  0.556839  \n",
      "2         tfidf_ng3  0.675338  0.556839  \n",
      "3         tfidf_ng3  0.675338  0.556839  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-KNN-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    18.0   51.0  318.0      NEG   0.260870  0.053571        KNN   \n",
      "1     3.0   10.0  163.0  NEUTRAL   0.230769  0.018072        KNN   \n",
      "2  1287.0  621.0   51.0      OBJ   0.674528  0.961883        KNN   \n",
      "3     2.0    7.0  157.0      POS   0.222222  0.012579        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.655328  0.550384  \n",
      "1         count_ng2  0.655328  0.550384  \n",
      "2         count_ng2  0.655328  0.550384  \n",
      "3         count_ng2  0.655328  0.550384  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  71.0   90.0   88.0      NEG   0.440994  0.446541  Passive Aggresive   \n",
      "1  57.0   90.0  102.0  NEUTRAL   0.387755  0.358491  Passive Aggresive   \n",
      "2  86.0  102.0   73.0      OBJ   0.457447  0.540881  Passive Aggresive   \n",
      "3  70.0   70.0   89.0      POS   0.500000  0.440252  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.446541  0.445051  \n",
      "1         tfidf_ng1  0.446541  0.445051  \n",
      "2         tfidf_ng1  0.446541  0.445051  \n",
      "3         tfidf_ng1  0.446541  0.445051  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall         Classifier  \\\n",
      "0  64.0   85.0  95.0      NEG   0.429530  0.402516  Passive Aggresive   \n",
      "1  67.0  109.0  92.0  NEUTRAL   0.380682  0.421384  Passive Aggresive   \n",
      "2  90.0   83.0  69.0      OBJ   0.520231  0.566038  Passive Aggresive   \n",
      "3  74.0   64.0  85.0      POS   0.536232  0.465409  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.463836  0.464017  \n",
      "1         tfidf_ng2  0.463836  0.464017  \n",
      "2         tfidf_ng2  0.463836  0.464017  \n",
      "3         tfidf_ng2  0.463836  0.464017  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-KNN-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    14.0   37.0  322.0      NEG   0.274510  0.041667        KNN   \n",
      "1     3.0   16.0  163.0  NEUTRAL   0.157895  0.018072        KNN   \n",
      "2  1284.0  625.0   54.0      OBJ   0.672603  0.959641        KNN   \n",
      "3     4.0   16.0  155.0      POS   0.200000  0.025157        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.652826  0.547775  \n",
      "1         count_ng1  0.652826  0.547775  \n",
      "2         count_ng1  0.652826  0.547775  \n",
      "3         count_ng1  0.652826  0.547775  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-bnb-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision  Recall Classifier  \\\n",
      "0     0.0    1.0  336.0      NEG    0.00000     1.0        bnb   \n",
      "1     0.0    0.0  166.0  NEUTRAL    0.00000     0.0        bnb   \n",
      "2  1338.0  660.0    0.0      OBJ    0.66967     1.0        bnb   \n",
      "3     0.0    0.0  159.0      POS    0.00000     0.0        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.669335  0.536912  \n",
      "1         tfidf_ng3  0.669335  0.536912  \n",
      "2         tfidf_ng3  0.669335  0.536912  \n",
      "3         tfidf_ng3  0.669335  0.536912  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-bnb-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision  Recall Classifier  \\\n",
      "0     0.0    1.0  336.0      NEG    0.00000     1.0        bnb   \n",
      "1     0.0    0.0  166.0  NEUTRAL    0.00000     0.0        bnb   \n",
      "2  1338.0  660.0    0.0      OBJ    0.66967     1.0        bnb   \n",
      "3     0.0    0.0  159.0      POS    0.00000     0.0        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.669335  0.536912  \n",
      "1         tfidf_ng2  0.669335  0.536912  \n",
      "2         tfidf_ng2  0.669335  0.536912  \n",
      "3         tfidf_ng2  0.669335  0.536912  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-bnb-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0     3.0    4.0  333.0      NEG   0.428571  0.008929        bnb   \n",
      "1     0.0    0.0  166.0  NEUTRAL   0.000000  0.000000        bnb   \n",
      "2  1336.0  656.0    2.0      OBJ   0.670683  0.998505        bnb   \n",
      "3     0.0    0.0  159.0      POS   0.000000  0.000000        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.669835  0.540016  \n",
      "1         tfidf_ng1  0.669835  0.540016  \n",
      "2         tfidf_ng1  0.669835  0.540016  \n",
      "3         tfidf_ng1  0.669835  0.540016  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Perceptron-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0   102.0  120.0  234.0      NEG   0.459459  0.303571  Perceptron   \n",
      "1    11.0   48.0  155.0  NEUTRAL   0.186441  0.066265  Perceptron   \n",
      "2  1201.0  445.0  137.0      OBJ   0.729648  0.897608  Perceptron   \n",
      "3    29.0   43.0  130.0      POS   0.402778  0.182390  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.671836  0.628328  \n",
      "1         count_ng2  0.671836  0.628328  \n",
      "2         count_ng2  0.671836  0.628328  \n",
      "3         count_ng2  0.671836  0.628328  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Perceptron-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0   106.0  123.0  230.0      NEG   0.462882  0.315476  Perceptron   \n",
      "1    19.0   63.0  147.0  NEUTRAL   0.231707  0.114458  Perceptron   \n",
      "2  1183.0  427.0  155.0      OBJ   0.734783  0.884155  Perceptron   \n",
      "3    31.0   47.0  128.0      POS   0.397436  0.194969  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.669835  0.633794  \n",
      "1         count_ng3  0.669835  0.633794  \n",
      "2         count_ng3  0.669835  0.633794  \n",
      "3         count_ng3  0.669835  0.633794  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Perceptron-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0   115.0  180.0  221.0      NEG   0.389831  0.342262  Perceptron   \n",
      "1    23.0  110.0  143.0  NEUTRAL   0.172932  0.138554  Perceptron   \n",
      "2  1121.0  385.0  217.0      OBJ   0.744356  0.837818  Perceptron   \n",
      "3    20.0   45.0  139.0      POS   0.307692  0.125786  Perceptron   \n",
      "\n",
      "  feauter_generator      acc      f1  \n",
      "0         count_ng1  0.63982  0.6159  \n",
      "1         count_ng1  0.63982  0.6159  \n",
      "2         count_ng1  0.63982  0.6159  \n",
      "3         count_ng1  0.63982  0.6159  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-mnb-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0     1.0    1.0  335.0      NEG   0.500000  0.002976        mnb   \n",
      "1     1.0    0.0  165.0  NEUTRAL   1.000000  0.006024        mnb   \n",
      "2  1338.0  658.0    0.0      OBJ   0.670341  1.000000        mnb   \n",
      "3     0.0    0.0  159.0      POS   0.000000  0.000000        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.670335  0.539224  \n",
      "1         tfidf_ng2  0.670335  0.539224  \n",
      "2         tfidf_ng2  0.670335  0.539224  \n",
      "3         tfidf_ng2  0.670335  0.539224  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-count_ng1\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   66.0   75.0   93.0      NEG   0.468085  0.415094        bnb   \n",
      "1   32.0   37.0  127.0  NEUTRAL   0.463768  0.201258        bnb   \n",
      "2   90.0   99.0   69.0      OBJ   0.476190  0.566038        bnb   \n",
      "3  109.0  128.0   50.0      POS   0.459916  0.685535        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.466981  0.447112  \n",
      "1         count_ng1  0.466981  0.447112  \n",
      "2         count_ng1  0.466981  0.447112  \n",
      "3         count_ng1  0.466981  0.447112  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-count_ng2\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   48.0   46.0  111.0      NEG   0.510638  0.301887        bnb   \n",
      "1   16.0    6.0  143.0  NEUTRAL   0.727273  0.100629        bnb   \n",
      "2   86.0   91.0   73.0      OBJ   0.485876  0.540881        bnb   \n",
      "3  124.0  219.0   35.0      POS   0.361516  0.779874        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.430818  0.390543  \n",
      "1         count_ng2  0.430818  0.390543  \n",
      "2         count_ng2  0.430818  0.390543  \n",
      "3         count_ng2  0.430818  0.390543  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-count_ng3\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   36.0   23.0  123.0      NEG   0.610169  0.226415        bnb   \n",
      "1    6.0    5.0  153.0  NEUTRAL   0.545455  0.037736        bnb   \n",
      "2   70.0   69.0   89.0      OBJ   0.503597  0.440252        bnb   \n",
      "3  139.0  288.0   20.0      POS   0.325527  0.874214        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.394654  0.336266  \n",
      "1         count_ng3  0.394654  0.336266  \n",
      "2         count_ng3  0.394654  0.336266  \n",
      "3         count_ng3  0.394654  0.336266  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-DecisionTree-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    82.0  173.0  254.0      NEG   0.321569  0.244048  DecisionTree   \n",
      "1    17.0  101.0  149.0  NEUTRAL   0.144068  0.102410  DecisionTree   \n",
      "2  1076.0  442.0  262.0      OBJ   0.708827  0.804185  DecisionTree   \n",
      "3    16.0   92.0  143.0      POS   0.148148  0.100629  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.595798  0.570462  \n",
      "1         tfidf_ng1  0.595798  0.570462  \n",
      "2         tfidf_ng1  0.595798  0.570462  \n",
      "3         tfidf_ng1  0.595798  0.570462  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-DecisionTree-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    76.0  195.0  260.0      NEG   0.280443  0.226190  DecisionTree   \n",
      "1    13.0   87.0  153.0  NEUTRAL   0.130000  0.078313  DecisionTree   \n",
      "2  1091.0  436.0  247.0      OBJ   0.714473  0.815396  DecisionTree   \n",
      "3    15.0   86.0  144.0      POS   0.148515  0.094340  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.597799  0.569154  \n",
      "1         tfidf_ng2  0.597799  0.569154  \n",
      "2         tfidf_ng2  0.597799  0.569154  \n",
      "3         tfidf_ng2  0.597799  0.569154  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-DecisionTree-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    82.0  214.0  254.0      NEG   0.277027  0.244048  DecisionTree   \n",
      "1    15.0   94.0  151.0  NEUTRAL   0.137615  0.090361  DecisionTree   \n",
      "2  1059.0  435.0  279.0      OBJ   0.708835  0.791480  DecisionTree   \n",
      "3    15.0   85.0  144.0      POS   0.150000  0.094340  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.585793  0.562472  \n",
      "1         tfidf_ng3  0.585793  0.562472  \n",
      "2         tfidf_ng3  0.585793  0.562472  \n",
      "3         tfidf_ng3  0.585793  0.562472  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   68.0  99.0      NEG   0.468750  0.377358        mnb   \n",
      "1  82.0  132.0  77.0  NEUTRAL   0.383178  0.515723        mnb   \n",
      "2  89.0   72.0  70.0      OBJ   0.552795  0.559748        mnb   \n",
      "3  77.0   56.0  82.0      POS   0.578947  0.484277        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.484277  0.485361  \n",
      "1         tfidf_ng3  0.484277  0.485361  \n",
      "2         tfidf_ng3  0.484277  0.485361  \n",
      "3         tfidf_ng3  0.484277  0.485361  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   69.0  99.0      NEG   0.465116  0.377358        mnb   \n",
      "1  83.0  130.0  76.0  NEUTRAL   0.389671  0.522013        mnb   \n",
      "2  90.0   72.0  69.0      OBJ   0.555556  0.566038        mnb   \n",
      "3  79.0   53.0  80.0      POS   0.598485  0.496855        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.490566  0.491652  \n",
      "1         tfidf_ng2  0.490566  0.491652  \n",
      "2         tfidf_ng2  0.490566  0.491652  \n",
      "3         tfidf_ng2  0.490566  0.491652  \n",
      "\n",
      "========================================\n",
      "4-balanced-mnb-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  67.0   80.0  92.0      NEG   0.455782  0.421384        mnb   \n",
      "1  79.0  128.0  80.0  NEUTRAL   0.381643  0.496855        mnb   \n",
      "2  86.0   68.0  73.0      OBJ   0.558442  0.540881        mnb   \n",
      "3  75.0   53.0  84.0      POS   0.585938  0.471698        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.482704  0.485443  \n",
      "1         tfidf_ng1  0.482704  0.485443  \n",
      "2         tfidf_ng1  0.482704  0.485443  \n",
      "3         tfidf_ng1  0.482704  0.485443  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  40.0   85.0  119.0      NEG   0.320000  0.251572  RandomForest   \n",
      "1  57.0  110.0  102.0  NEUTRAL   0.341317  0.358491  RandomForest   \n",
      "2  66.0   70.0   93.0      OBJ   0.485294  0.415094  RandomForest   \n",
      "3  88.0  120.0   71.0      POS   0.423077  0.553459  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.394654  0.389601  \n",
      "1         tfidf_ng1  0.394654  0.389601  \n",
      "2         tfidf_ng1  0.394654  0.389601  \n",
      "3         tfidf_ng1  0.394654  0.389601  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall    Classifier  \\\n",
      "0  62.0  103.0  97.0      NEG   0.375758  0.389937  RandomForest   \n",
      "1  68.0  105.0  91.0  NEUTRAL   0.393064  0.427673  RandomForest   \n",
      "2  60.0   73.0  99.0      OBJ   0.451128  0.377358  RandomForest   \n",
      "3  76.0   89.0  83.0      POS   0.460606  0.477987  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.418239  0.418112  \n",
      "1         tfidf_ng3  0.418239  0.418112  \n",
      "2         tfidf_ng3  0.418239  0.418112  \n",
      "3         tfidf_ng3  0.418239  0.418112  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  45.0   83.0  114.0      NEG   0.351562  0.283019  RandomForest   \n",
      "1  64.0   92.0   95.0  NEUTRAL   0.410256  0.402516  RandomForest   \n",
      "2  73.0   90.0   86.0      OBJ   0.447853  0.459119  RandomForest   \n",
      "3  77.0  112.0   82.0      POS   0.407407  0.484277  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.407233  0.403971  \n",
      "1         tfidf_ng2  0.407233  0.403971  \n",
      "2         tfidf_ng2  0.407233  0.403971  \n",
      "3         tfidf_ng2  0.407233  0.403971  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  54.0  106.0  105.0      NEG   0.337500  0.339623  DecisionTree   \n",
      "1  48.0   99.0  111.0  NEUTRAL   0.326531  0.301887  DecisionTree   \n",
      "2  69.0  105.0   90.0      OBJ   0.396552  0.433962  DecisionTree   \n",
      "3  65.0   90.0   94.0      POS   0.419355  0.408805  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.371069  0.370178  \n",
      "1         count_ng1  0.371069  0.370178  \n",
      "2         count_ng1  0.371069  0.370178  \n",
      "3         count_ng1  0.371069  0.370178  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  58.0  122.0  101.0      NEG   0.322222  0.364780  DecisionTree   \n",
      "1  48.0   94.0  111.0  NEUTRAL   0.338028  0.301887  DecisionTree   \n",
      "2  62.0  100.0   97.0      OBJ   0.382716  0.389937  DecisionTree   \n",
      "3  62.0   90.0   97.0      POS   0.407895  0.389937  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.361635  0.361532  \n",
      "1         count_ng3  0.361635  0.361532  \n",
      "2         count_ng3  0.361635  0.361532  \n",
      "3         count_ng3  0.361635  0.361532  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  54.0  111.0  105.0      NEG   0.327273  0.339623  DecisionTree   \n",
      "1  50.0   94.0  109.0  NEUTRAL   0.347222  0.314465  DecisionTree   \n",
      "2  66.0  120.0   93.0      OBJ   0.354839  0.415094  DecisionTree   \n",
      "3  60.0   81.0   99.0      POS   0.425532  0.377358  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.361635  0.361494  \n",
      "1         count_ng2  0.361635  0.361494  \n",
      "2         count_ng2  0.361635  0.361494  \n",
      "3         count_ng2  0.361635  0.361494  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-RandomForest-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    15.0   14.0  321.0      NEG   0.517241  0.044643  RandomForest   \n",
      "1     1.0    0.0  165.0  NEUTRAL   1.000000  0.006024  RandomForest   \n",
      "2  1327.0  640.0   11.0      OBJ   0.674631  0.991779  RandomForest   \n",
      "3     2.0    0.0  157.0      POS   1.000000  0.012579  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.672836  0.554279  \n",
      "1         tfidf_ng1  0.672836  0.554279  \n",
      "2         tfidf_ng1  0.672836  0.554279  \n",
      "3         tfidf_ng1  0.672836  0.554279  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-RandomForest-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    13.0   10.0  323.0      NEG   0.565217  0.038690  RandomForest   \n",
      "1     2.0    9.0  164.0  NEUTRAL   0.181818  0.012048  RandomForest   \n",
      "2  1330.0  632.0    8.0      OBJ   0.677880  0.994021  RandomForest   \n",
      "3     3.0    0.0  156.0      POS   1.000000  0.018868  RandomForest   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         tfidf_ng2  0.674337  0.55652  \n",
      "1         tfidf_ng2  0.674337  0.55652  \n",
      "2         tfidf_ng2  0.674337  0.55652  \n",
      "3         tfidf_ng2  0.674337  0.55652  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-RandomForest-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    10.0    6.0  326.0      NEG   0.625000  0.029762  RandomForest   \n",
      "1     1.0    2.0  165.0  NEUTRAL   0.333333  0.006024  RandomForest   \n",
      "2  1332.0  645.0    6.0      OBJ   0.673748  0.995516  RandomForest   \n",
      "3     2.0    1.0  157.0      POS   0.666667  0.012579  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.672836  0.550388  \n",
      "1         tfidf_ng3  0.672836  0.550388  \n",
      "2         tfidf_ng3  0.672836  0.550388  \n",
      "3         tfidf_ng3  0.672836  0.550388  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-SVM-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    98.0  156.0  238.0      NEG   0.385827  0.291667        SVM   \n",
      "1    17.0   66.0  149.0  NEUTRAL   0.204819  0.102410        SVM   \n",
      "2  1161.0  423.0  177.0      OBJ   0.732955  0.867713        SVM   \n",
      "3    25.0   53.0  134.0      POS   0.320513  0.157233        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.650825  0.615852  \n",
      "1         count_ng1  0.650825  0.615852  \n",
      "2         count_ng1  0.650825  0.615852  \n",
      "3         count_ng1  0.650825  0.615852  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-SVM-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    79.0   85.0  257.0      NEG   0.481707  0.235119        SVM   \n",
      "1     8.0   23.0  158.0  NEUTRAL   0.258065  0.048193        SVM   \n",
      "2  1261.0  505.0   77.0      OBJ   0.714043  0.942451        SVM   \n",
      "3    16.0   22.0  143.0      POS   0.421053  0.100629        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.682341  0.616614  \n",
      "1         count_ng3  0.682341  0.616614  \n",
      "2         count_ng3  0.682341  0.616614  \n",
      "3         count_ng3  0.682341  0.616614  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-SVM-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    90.0  102.0  246.0      NEG   0.468750  0.267857        SVM   \n",
      "1    10.0   39.0  156.0  NEUTRAL   0.204082  0.060241        SVM   \n",
      "2  1243.0  468.0   95.0      OBJ   0.726476  0.928999        SVM   \n",
      "3    20.0   27.0  139.0      POS   0.425532  0.125786        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.681841  0.626212  \n",
      "1         count_ng2  0.681841  0.626212  \n",
      "2         count_ng2  0.681841  0.626212  \n",
      "3         count_ng2  0.681841  0.626212  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  87.0  109.0   72.0      NEG   0.443878  0.547170        sgd   \n",
      "1  41.0   80.0  118.0  NEUTRAL   0.338843  0.257862        sgd   \n",
      "2  61.0   82.0   98.0      OBJ   0.426573  0.383648        sgd   \n",
      "3  80.0   96.0   79.0      POS   0.454545  0.503145        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.422956  0.416146  \n",
      "1         count_ng1  0.422956  0.416146  \n",
      "2         count_ng1  0.422956  0.416146  \n",
      "3         count_ng1  0.422956  0.416146  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  56.0   92.0  103.0      NEG   0.378378  0.352201        sgd   \n",
      "1  54.0  100.0  105.0  NEUTRAL   0.350649  0.339623        sgd   \n",
      "2  75.0   71.0   84.0      OBJ   0.513699  0.471698        sgd   \n",
      "3  92.0   96.0   67.0      POS   0.489362  0.578616        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.435535  0.432983  \n",
      "1         count_ng2  0.435535  0.432983  \n",
      "2         count_ng2  0.435535  0.432983  \n",
      "3         count_ng2  0.435535  0.432983  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   86.0  94.0      NEG   0.430464  0.408805        sgd   \n",
      "1  64.0  108.0  95.0  NEUTRAL   0.372093  0.402516        sgd   \n",
      "2  73.0   75.0  86.0      OBJ   0.493243  0.459119        sgd   \n",
      "3  85.0   80.0  74.0      POS   0.515152  0.534591        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.451258  0.451581  \n",
      "1         count_ng3  0.451258  0.451581  \n",
      "2         count_ng3  0.451258  0.451581  \n",
      "3         count_ng3  0.451258  0.451581  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  49.0   82.0  110.0      NEG   0.374046  0.308176  RandomForest   \n",
      "1  55.0  103.0  104.0  NEUTRAL   0.348101  0.345912  RandomForest   \n",
      "2  68.0   76.0   91.0      OBJ   0.472222  0.427673  RandomForest   \n",
      "3  95.0  108.0   64.0      POS   0.467980  0.597484  RandomForest   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng1  0.419811  0.41466  \n",
      "1         count_ng1  0.419811  0.41466  \n",
      "2         count_ng1  0.419811  0.41466  \n",
      "3         count_ng1  0.419811  0.41466  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  50.0   77.0  109.0      NEG   0.393701  0.314465  RandomForest   \n",
      "1  56.0  101.0  103.0  NEUTRAL   0.356688  0.352201  RandomForest   \n",
      "2  56.0   63.0  103.0      OBJ   0.470588  0.352201  RandomForest   \n",
      "3  94.0  139.0   65.0      POS   0.403433  0.591195  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.402516  0.396638  \n",
      "1         count_ng3  0.402516  0.396638  \n",
      "2         count_ng3  0.402516  0.396638  \n",
      "3         count_ng3  0.402516  0.396638  \n",
      "\n",
      "========================================\n",
      "4-balanced-RandomForest-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  56.0   88.0  103.0      NEG   0.388889  0.352201  RandomForest   \n",
      "1  58.0   92.0  101.0  NEUTRAL   0.386667  0.364780  RandomForest   \n",
      "2  58.0   54.0  101.0      OBJ   0.517857  0.364780  RandomForest   \n",
      "3  99.0  131.0   60.0      POS   0.430435  0.622642  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.426101  0.420521  \n",
      "1         count_ng2  0.426101  0.420521  \n",
      "2         count_ng2  0.426101  0.420521  \n",
      "3         count_ng2  0.426101  0.420521  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Perceptron-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0   117.0  203.0  219.0      NEG   0.365625  0.348214  Perceptron   \n",
      "1    18.0   78.0  148.0  NEUTRAL   0.187500  0.108434  Perceptron   \n",
      "2  1112.0  356.0  226.0      OBJ   0.757493  0.831091  Perceptron   \n",
      "3    35.0   80.0  124.0      POS   0.304348  0.220126  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.641321  0.622194  \n",
      "1         tfidf_ng2  0.641321  0.622194  \n",
      "2         tfidf_ng2  0.641321  0.622194  \n",
      "3         tfidf_ng2  0.641321  0.622194  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Perceptron-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0   114.0  191.0  222.0      NEG   0.373770  0.339286  Perceptron   \n",
      "1    17.0   81.0  149.0  NEUTRAL   0.173469  0.102410  Perceptron   \n",
      "2  1103.0  369.0  235.0      OBJ   0.749321  0.824365  Perceptron   \n",
      "3    34.0   90.0  125.0      POS   0.274194  0.213836  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.634317  0.615057  \n",
      "1         tfidf_ng3  0.634317  0.615057  \n",
      "2         tfidf_ng3  0.634317  0.615057  \n",
      "3         tfidf_ng3  0.634317  0.615057  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Perceptron-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0    95.0  179.0  241.0      NEG   0.346715  0.282738  Perceptron   \n",
      "1    20.0   95.0  146.0  NEUTRAL   0.173913  0.120482  Perceptron   \n",
      "2  1110.0  393.0  228.0      OBJ   0.738523  0.829596  Perceptron   \n",
      "3    34.0   73.0  125.0      POS   0.317757  0.213836  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.629815  0.607537  \n",
      "1         tfidf_ng1  0.629815  0.607537  \n",
      "2         tfidf_ng1  0.629815  0.607537  \n",
      "3         tfidf_ng1  0.629815  0.607537  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-tfidf_ng3\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   36.0   23.0  123.0      NEG   0.610169  0.226415        bnb   \n",
      "1    6.0    5.0  153.0  NEUTRAL   0.545455  0.037736        bnb   \n",
      "2   70.0   69.0   89.0      OBJ   0.503597  0.440252        bnb   \n",
      "3  139.0  288.0   20.0      POS   0.325527  0.874214        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.394654  0.336266  \n",
      "1         tfidf_ng3  0.394654  0.336266  \n",
      "2         tfidf_ng3  0.394654  0.336266  \n",
      "3         tfidf_ng3  0.394654  0.336266  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-RandomForest-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    15.0   10.0  321.0      NEG   0.600000  0.044643  RandomForest   \n",
      "1     1.0    0.0  165.0  NEUTRAL   1.000000  0.006024  RandomForest   \n",
      "2  1329.0  644.0    9.0      OBJ   0.673594  0.993274  RandomForest   \n",
      "3     0.0    0.0  159.0      POS   0.000000  0.000000  RandomForest   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng1  0.672836  0.55229  \n",
      "1         count_ng1  0.672836  0.55229  \n",
      "2         count_ng1  0.672836  0.55229  \n",
      "3         count_ng1  0.672836  0.55229  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-RandomForest-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0     8.0    6.0  328.0      NEG   0.571429  0.023810  RandomForest   \n",
      "1     1.0    0.0  165.0  NEUTRAL   1.000000  0.006024  RandomForest   \n",
      "2  1334.0  650.0    4.0      OBJ   0.672379  0.997010  RandomForest   \n",
      "3     0.0    0.0  159.0      POS   0.000000  0.000000  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.671836  0.546242  \n",
      "1         count_ng2  0.671836  0.546242  \n",
      "2         count_ng2  0.671836  0.546242  \n",
      "3         count_ng2  0.671836  0.546242  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-RandomForest-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    10.0    2.0  326.0      NEG   0.833333  0.029762  RandomForest   \n",
      "1     1.0    2.0  165.0  NEUTRAL   0.333333  0.006024  RandomForest   \n",
      "2  1336.0  646.0    2.0      OBJ   0.674067  0.998505  RandomForest   \n",
      "3     2.0    0.0  157.0      POS   1.000000  0.012579  RandomForest   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.674837  0.551312  \n",
      "1         count_ng3  0.674837  0.551312  \n",
      "2         count_ng3  0.674837  0.551312  \n",
      "3         count_ng3  0.674837  0.551312  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-sgd-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    87.0  106.0  249.0      NEG   0.450777  0.258929        sgd   \n",
      "1    11.0   41.0  155.0  NEUTRAL   0.211538  0.066265        sgd   \n",
      "2  1235.0  477.0  103.0      OBJ   0.721379  0.923019        sgd   \n",
      "3    22.0   20.0  137.0      POS   0.523810  0.138365        sgd   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng3  0.677839  0.62313  \n",
      "1         count_ng3  0.677839  0.62313  \n",
      "2         count_ng3  0.677839  0.62313  \n",
      "3         count_ng3  0.677839  0.62313  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-sgd-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    88.0  102.0  248.0      NEG   0.463158  0.261905        sgd   \n",
      "1    16.0   54.0  150.0  NEUTRAL   0.228571  0.096386        sgd   \n",
      "2  1226.0  466.0  112.0      OBJ   0.724586  0.916293        sgd   \n",
      "3    23.0   24.0  136.0      POS   0.489362  0.144654        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.676838  0.626915  \n",
      "1         count_ng2  0.676838  0.626915  \n",
      "2         count_ng2  0.676838  0.626915  \n",
      "3         count_ng2  0.676838  0.626915  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-sgd-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    81.0  132.0  255.0      NEG   0.380282  0.241071        sgd   \n",
      "1    14.0   73.0  152.0  NEUTRAL   0.160920  0.084337        sgd   \n",
      "2  1197.0  451.0  141.0      OBJ   0.726335  0.894619        sgd   \n",
      "3    21.0   30.0  138.0      POS   0.411765  0.132075        sgd   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng1  0.656828  0.61133  \n",
      "1         count_ng1  0.656828  0.61133  \n",
      "2         count_ng1  0.656828  0.61133  \n",
      "3         count_ng1  0.656828  0.61133  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Passive Aggresive-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0   119.0  144.0  217.0      NEG   0.452471  0.354167  Passive Aggresive   \n",
      "1    15.0   60.0  151.0  NEUTRAL   0.200000  0.090361  Passive Aggresive   \n",
      "2  1190.0  412.0  148.0      OBJ   0.742821  0.889387  Passive Aggresive   \n",
      "3    29.0   30.0  130.0      POS   0.491525  0.182390  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.676838  0.640126  \n",
      "1         tfidf_ng3  0.676838  0.640126  \n",
      "2         tfidf_ng3  0.676838  0.640126  \n",
      "3         tfidf_ng3  0.676838  0.640126  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Passive Aggresive-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0   121.0  155.0  215.0      NEG   0.438406  0.360119  Passive Aggresive   \n",
      "1    16.0   65.0  150.0  NEUTRAL   0.197531  0.096386  Passive Aggresive   \n",
      "2  1180.0  401.0  158.0      OBJ   0.746363  0.881913  Passive Aggresive   \n",
      "3    30.0   31.0  129.0      POS   0.491803  0.188679  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         tfidf_ng2  0.673837  0.64007  \n",
      "1         tfidf_ng2  0.673837  0.64007  \n",
      "2         tfidf_ng2  0.673837  0.64007  \n",
      "3         tfidf_ng2  0.673837  0.64007  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Passive Aggresive-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0   107.0  184.0  229.0      NEG   0.367698  0.318452  Passive Aggresive   \n",
      "1    16.0   83.0  150.0  NEUTRAL   0.161616  0.096386  Passive Aggresive   \n",
      "2  1128.0  403.0  210.0      OBJ   0.736773  0.843049  Passive Aggresive   \n",
      "3    31.0   47.0  128.0      POS   0.397436  0.194969  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.641321  0.614526  \n",
      "1         tfidf_ng1  0.641321  0.614526  \n",
      "2         tfidf_ng1  0.641321  0.614526  \n",
      "3         tfidf_ng1  0.641321  0.614526  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-KNN-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    89.0  124.0  247.0      NEG   0.417840  0.264881        KNN   \n",
      "1    16.0   41.0  150.0  NEUTRAL   0.280702  0.096386        KNN   \n",
      "2  1212.0  478.0  126.0      OBJ   0.717160  0.905830        KNN   \n",
      "3    17.0   22.0  142.0      POS   0.435897  0.106918        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.667334  0.615893  \n",
      "1         tfidf_ng3  0.667334  0.615893  \n",
      "2         tfidf_ng3  0.667334  0.615893  \n",
      "3         tfidf_ng3  0.667334  0.615893  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-mnb-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    83.0   75.0  253.0      NEG   0.525316  0.247024        mnb   \n",
      "1     1.0    8.0  165.0  NEUTRAL   0.111111  0.006024        mnb   \n",
      "2  1285.0  539.0   53.0      OBJ   0.704496  0.960389        mnb   \n",
      "3     3.0    5.0  156.0      POS   0.375000  0.018868        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.686343  0.604308  \n",
      "1         count_ng1  0.686343  0.604308  \n",
      "2         count_ng1  0.686343  0.604308  \n",
      "3         count_ng1  0.686343  0.604308  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-mnb-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    50.0   40.0  286.0      NEG   0.555556  0.148810        mnb   \n",
      "1     3.0    8.0  163.0  NEUTRAL   0.272727  0.018072        mnb   \n",
      "2  1311.0  579.0   27.0      OBJ   0.693651  0.979821        mnb   \n",
      "3     5.0    3.0  154.0      POS   0.625000  0.031447        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.684842  0.590713  \n",
      "1         count_ng2  0.684842  0.590713  \n",
      "2         count_ng2  0.684842  0.590713  \n",
      "3         count_ng2  0.684842  0.590713  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-mnb-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    41.0   32.0  295.0      NEG   0.561644  0.122024        mnb   \n",
      "1     3.0    8.0  163.0  NEUTRAL   0.272727  0.018072        mnb   \n",
      "2  1315.0  592.0   23.0      OBJ   0.689565  0.982810        mnb   \n",
      "3     5.0    3.0  154.0      POS   0.625000  0.031447        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.682341  0.583758  \n",
      "1         count_ng3  0.682341  0.583758  \n",
      "2         count_ng3  0.682341  0.583758  \n",
      "3         count_ng3  0.682341  0.583758  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  51.0  102.0  108.0      NEG   0.333333  0.320755  DecisionTree   \n",
      "1  51.0   87.0  108.0  NEUTRAL   0.369565  0.320755  DecisionTree   \n",
      "2  59.0  108.0  100.0      OBJ   0.353293  0.371069  DecisionTree   \n",
      "3  62.0  116.0   97.0      POS   0.348315  0.389937  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.350629  0.350068  \n",
      "1         tfidf_ng1  0.350629  0.350068  \n",
      "2         tfidf_ng1  0.350629  0.350068  \n",
      "3         tfidf_ng1  0.350629  0.350068  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  43.0   68.0  116.0      NEG   0.387387  0.270440  DecisionTree   \n",
      "1  50.0   88.0  109.0  NEUTRAL   0.362319  0.314465  DecisionTree   \n",
      "2  81.0  124.0   78.0      OBJ   0.395122  0.509434  DecisionTree   \n",
      "3  68.0  114.0   91.0      POS   0.373626  0.427673  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.380503  0.374775  \n",
      "1         tfidf_ng3  0.380503  0.374775  \n",
      "2         tfidf_ng3  0.380503  0.374775  \n",
      "3         tfidf_ng3  0.380503  0.374775  \n",
      "\n",
      "========================================\n",
      "4-balanced-DecisionTree-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0  40.0   78.0  119.0      NEG   0.338983  0.251572  DecisionTree   \n",
      "1  44.0   93.0  115.0  NEUTRAL   0.321168  0.276730  DecisionTree   \n",
      "2  69.0  118.0   90.0      OBJ   0.368984  0.433962  DecisionTree   \n",
      "3  71.0  123.0   88.0      POS   0.365979  0.446541  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.352201  0.346804  \n",
      "1         tfidf_ng2  0.352201  0.346804  \n",
      "2         tfidf_ng2  0.352201  0.346804  \n",
      "3         tfidf_ng2  0.352201  0.346804  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  59.0   92.0  100.0      NEG   0.390728  0.371069  Logistic Regression   \n",
      "1  53.0   74.0  106.0  NEUTRAL   0.417323  0.333333  Logistic Regression   \n",
      "2  71.0   74.0   88.0      OBJ   0.489655  0.446541  Logistic Regression   \n",
      "3  96.0  117.0   63.0      POS   0.450704  0.603774  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.438679  0.433627  \n",
      "1         count_ng3  0.438679  0.433627  \n",
      "2         count_ng3  0.438679  0.433627  \n",
      "3         count_ng3  0.438679  0.433627  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  62.0  175.0   97.0      NEG   0.261603  0.389937        KNN   \n",
      "1   6.0   20.0  153.0  NEUTRAL   0.230769  0.037736        KNN   \n",
      "2  59.0  124.0  100.0      OBJ   0.322404  0.371069        KNN   \n",
      "3  56.0  134.0  103.0      POS   0.294737  0.352201        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.287736  0.260986  \n",
      "1         count_ng1  0.287736  0.260986  \n",
      "2         count_ng1  0.287736  0.260986  \n",
      "3         count_ng1  0.287736  0.260986  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-sgd-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    96.0  107.0  240.0      NEG   0.472906  0.285714        sgd   \n",
      "1     8.0   27.0  158.0  NEUTRAL   0.228571  0.048193        sgd   \n",
      "2  1258.0  467.0   80.0      OBJ   0.729275  0.940209        sgd   \n",
      "3    20.0   16.0  139.0      POS   0.555556  0.125786        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.691346  0.632603  \n",
      "1         tfidf_ng2  0.691346  0.632603  \n",
      "2         tfidf_ng2  0.691346  0.632603  \n",
      "3         tfidf_ng2  0.691346  0.632603  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  89.0  225.0   70.0      NEG   0.283439  0.559748        KNN   \n",
      "1   3.0    4.0  156.0  NEUTRAL   0.428571  0.018868        KNN   \n",
      "2  50.0  132.0  109.0      OBJ   0.274725  0.314465        KNN   \n",
      "3  33.0  100.0  126.0      POS   0.248120  0.207547        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.275157  0.232937  \n",
      "1         count_ng3  0.275157  0.232937  \n",
      "2         count_ng3  0.275157  0.232937  \n",
      "3         count_ng3  0.275157  0.232937  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  62.0   97.0   97.0      NEG   0.389937  0.389937  Logistic Regression   \n",
      "1  53.0   74.0  106.0  NEUTRAL   0.417323  0.333333  Logistic Regression   \n",
      "2  72.0   73.0   87.0      OBJ   0.496552  0.452830  Logistic Regression   \n",
      "3  96.0  109.0   63.0      POS   0.468293  0.603774  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.444969  0.440431  \n",
      "1         count_ng2  0.444969  0.440431  \n",
      "2         count_ng2  0.444969  0.440431  \n",
      "3         count_ng2  0.444969  0.440431  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  65.0   83.0   94.0      NEG   0.439189  0.408805  Passive Aggresive   \n",
      "1  49.0   82.0  110.0  NEUTRAL   0.374046  0.308176  Passive Aggresive   \n",
      "2  88.0  119.0   71.0      OBJ   0.425121  0.553459  Passive Aggresive   \n",
      "3  72.0   78.0   87.0      POS   0.480000  0.452830  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.430818  0.427069  \n",
      "1         count_ng1  0.430818  0.427069  \n",
      "2         count_ng1  0.430818  0.427069  \n",
      "3         count_ng1  0.430818  0.427069  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  64.0   92.0   95.0      NEG   0.410256  0.402516  Passive Aggresive   \n",
      "1  51.0   65.0  108.0  NEUTRAL   0.439655  0.320755  Passive Aggresive   \n",
      "2  81.0   98.0   78.0      OBJ   0.452514  0.509434  Passive Aggresive   \n",
      "3  83.0  102.0   76.0      POS   0.448649  0.522013  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.438679  0.434777  \n",
      "1         count_ng2  0.438679  0.434777  \n",
      "2         count_ng2  0.438679  0.434777  \n",
      "3         count_ng2  0.438679  0.434777  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0  65.0   90.0   94.0      NEG   0.419355  0.408805  Passive Aggresive   \n",
      "1  52.0   61.0  107.0  NEUTRAL   0.460177  0.327044  Passive Aggresive   \n",
      "2  82.0   99.0   77.0      OBJ   0.453039  0.515723  Passive Aggresive   \n",
      "3  85.0  102.0   74.0      POS   0.454545  0.534591  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.446541  0.442512  \n",
      "1         count_ng3  0.446541  0.442512  \n",
      "2         count_ng3  0.446541  0.442512  \n",
      "3         count_ng3  0.446541  0.442512  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  61.0  107.0   98.0      NEG   0.363095  0.383648        KNN   \n",
      "1  48.0  123.0  111.0  NEUTRAL   0.280702  0.301887        KNN   \n",
      "2  77.0  103.0   82.0      OBJ   0.427778  0.484277        KNN   \n",
      "3  49.0   68.0  110.0      POS   0.418803  0.308176        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.369497  0.368337  \n",
      "1         tfidf_ng1  0.369497  0.368337  \n",
      "2         tfidf_ng1  0.369497  0.368337  \n",
      "3         tfidf_ng1  0.369497  0.368337  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  63.0  101.0   96.0      NEG   0.384146  0.396226        KNN   \n",
      "1  58.0  116.0  101.0  NEUTRAL   0.333333  0.364780        KNN   \n",
      "2  82.0   85.0   77.0      OBJ   0.491018  0.515723        KNN   \n",
      "3  58.0   73.0  101.0      POS   0.442748  0.364780        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.410377  0.410377  \n",
      "1         tfidf_ng3  0.410377  0.410377  \n",
      "2         tfidf_ng3  0.410377  0.410377  \n",
      "3         tfidf_ng3  0.410377  0.410377  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  63.0  101.0   96.0      NEG   0.384146  0.396226        KNN   \n",
      "1  59.0  119.0  100.0  NEUTRAL   0.331461  0.371069        KNN   \n",
      "2  80.0   87.0   79.0      OBJ   0.479042  0.503145        KNN   \n",
      "3  58.0   69.0  101.0      POS   0.456693  0.364780        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.408805  0.409158  \n",
      "1         tfidf_ng2  0.408805  0.409158  \n",
      "2         tfidf_ng2  0.408805  0.409158  \n",
      "3         tfidf_ng2  0.408805  0.409158  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-KNN-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    84.0  102.0  252.0      NEG   0.451613  0.250000        KNN   \n",
      "1     7.0   43.0  159.0  NEUTRAL   0.140000  0.042169        KNN   \n",
      "2  1230.0  495.0  108.0      OBJ   0.713043  0.919283        KNN   \n",
      "3    13.0   25.0  146.0      POS   0.342105  0.081761        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.667334  0.607542  \n",
      "1         tfidf_ng1  0.667334  0.607542  \n",
      "2         tfidf_ng1  0.667334  0.607542  \n",
      "3         tfidf_ng1  0.667334  0.607542  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-KNN-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    15.0   39.0  321.0      NEG   0.277778  0.044643        KNN   \n",
      "1     2.0    5.0  164.0  NEUTRAL   0.285714  0.012048        KNN   \n",
      "2  1298.0  633.0   40.0      OBJ   0.672191  0.970105        KNN   \n",
      "3     0.0    7.0  159.0      POS   0.000000  1.000000        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.657829  0.546386  \n",
      "1         count_ng3  0.657829  0.546386  \n",
      "2         count_ng3  0.657829  0.546386  \n",
      "3         count_ng3  0.657829  0.546386  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-tfidf_ng1\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   66.0   75.0   93.0      NEG   0.468085  0.415094        bnb   \n",
      "1   32.0   37.0  127.0  NEUTRAL   0.463768  0.201258        bnb   \n",
      "2   90.0   99.0   69.0      OBJ   0.476190  0.566038        bnb   \n",
      "3  109.0  128.0   50.0      POS   0.459916  0.685535        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.466981  0.447112  \n",
      "1         tfidf_ng1  0.466981  0.447112  \n",
      "2         tfidf_ng1  0.466981  0.447112  \n",
      "3         tfidf_ng1  0.466981  0.447112  \n",
      "\n",
      "========================================\n",
      "4-balanced-bnb-tfidf_ng2\n",
      "----------------------------------------\n",
      "      TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0   48.0   46.0  111.0      NEG   0.510638  0.301887        bnb   \n",
      "1   16.0    6.0  143.0  NEUTRAL   0.727273  0.100629        bnb   \n",
      "2   86.0   91.0   73.0      OBJ   0.485876  0.540881        bnb   \n",
      "3  124.0  219.0   35.0      POS   0.361516  0.779874        bnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.430818  0.390543  \n",
      "1         tfidf_ng2  0.430818  0.390543  \n",
      "2         tfidf_ng2  0.430818  0.390543  \n",
      "3         tfidf_ng2  0.430818  0.390543  \n",
      "\n",
      "========================================\n",
      "4-balanced-Passive Aggresive-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall         Classifier  \\\n",
      "0  60.0   82.0  99.0      NEG   0.422535  0.377358  Passive Aggresive   \n",
      "1  71.0  112.0  88.0  NEUTRAL   0.387978  0.446541  Passive Aggresive   \n",
      "2  85.0   91.0  74.0      OBJ   0.482955  0.534591  Passive Aggresive   \n",
      "3  70.0   65.0  89.0      POS   0.518519  0.440252  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.449686  0.449382  \n",
      "1         tfidf_ng3  0.449686  0.449382  \n",
      "2         tfidf_ng3  0.449686  0.449382  \n",
      "3         tfidf_ng3  0.449686  0.449382  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  69.0   82.0   90.0      NEG   0.456954  0.433962  Perceptron   \n",
      "1  57.0   91.0  102.0  NEUTRAL   0.385135  0.358491  Perceptron   \n",
      "2  82.0  101.0   77.0      OBJ   0.448087  0.515723  Perceptron   \n",
      "3  73.0   81.0   86.0      POS   0.474026  0.459119  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.441824  0.440621  \n",
      "1         tfidf_ng3  0.441824  0.440621  \n",
      "2         tfidf_ng3  0.441824  0.440621  \n",
      "3         tfidf_ng3  0.441824  0.440621  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP    FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  71.0  95.0   88.0      NEG   0.427711  0.446541  Perceptron   \n",
      "1  52.0  83.0  107.0  NEUTRAL   0.385185  0.327044  Perceptron   \n",
      "2  77.0  82.0   82.0      OBJ   0.484277  0.484277  Perceptron   \n",
      "3  85.0  91.0   74.0      POS   0.482955  0.534591  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.448113  0.445601  \n",
      "1         tfidf_ng2  0.448113  0.445601  \n",
      "2         tfidf_ng2  0.448113  0.445601  \n",
      "3         tfidf_ng2  0.448113  0.445601  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  73.0   96.0   86.0      NEG   0.431953  0.459119  Perceptron   \n",
      "1  46.0   91.0  113.0  NEUTRAL   0.335766  0.289308  Perceptron   \n",
      "2  90.0  114.0   69.0      OBJ   0.441176  0.566038  Perceptron   \n",
      "3  64.0   62.0   95.0      POS   0.507937  0.402516  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.429245  0.425231  \n",
      "1         tfidf_ng1  0.429245  0.425231  \n",
      "2         tfidf_ng1  0.429245  0.425231  \n",
      "3         tfidf_ng1  0.429245  0.425231  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-KNN-tfidf_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    89.0  126.0  247.0      NEG   0.413953  0.264881        KNN   \n",
      "1    14.0   43.0  152.0  NEUTRAL   0.245614  0.084337        KNN   \n",
      "2  1212.0  479.0  126.0      OBJ   0.716736  0.905830        KNN   \n",
      "3    16.0   20.0  143.0      POS   0.444444  0.100629        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.665833  0.613423  \n",
      "1         tfidf_ng2  0.665833  0.613423  \n",
      "2         tfidf_ng2  0.665833  0.613423  \n",
      "3         tfidf_ng2  0.665833  0.613423  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-DecisionTree-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    90.0  174.0  246.0      NEG   0.340909  0.267857  DecisionTree   \n",
      "1    13.0   80.0  153.0  NEUTRAL   0.139785  0.078313  DecisionTree   \n",
      "2  1120.0  450.0  218.0      OBJ   0.713376  0.837070  DecisionTree   \n",
      "3    18.0   54.0  141.0      POS   0.250000  0.113208  DecisionTree   \n",
      "\n",
      "  feauter_generator      acc        f1  \n",
      "0         count_ng1  0.62081  0.586738  \n",
      "1         count_ng1  0.62081  0.586738  \n",
      "2         count_ng1  0.62081  0.586738  \n",
      "3         count_ng1  0.62081  0.586738  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-DecisionTree-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    87.0  180.0  249.0      NEG   0.325843  0.258929  DecisionTree   \n",
      "1     9.0   71.0  157.0  NEUTRAL   0.112500  0.054217  DecisionTree   \n",
      "2  1137.0  455.0  201.0      OBJ   0.714196  0.849776  DecisionTree   \n",
      "3    15.0   45.0  144.0      POS   0.250000  0.094340  DecisionTree   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.624312  0.584951  \n",
      "1         count_ng2  0.624312  0.584951  \n",
      "2         count_ng2  0.624312  0.584951  \n",
      "3         count_ng2  0.624312  0.584951  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-DecisionTree-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall    Classifier  \\\n",
      "0    89.0  192.0  247.0      NEG   0.316726  0.264881  DecisionTree   \n",
      "1    11.0   67.0  155.0  NEUTRAL   0.141026  0.066265  DecisionTree   \n",
      "2  1122.0  450.0  216.0      OBJ   0.713740  0.838565  DecisionTree   \n",
      "3    17.0   51.0  142.0      POS   0.250000  0.106918  DecisionTree   \n",
      "\n",
      "  feauter_generator      acc        f1  \n",
      "0         count_ng3  0.61981  0.584039  \n",
      "1         count_ng3  0.61981  0.584039  \n",
      "2         count_ng3  0.61981  0.584039  \n",
      "3         count_ng3  0.61981  0.584039  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  63.0   89.0   96.0      NEG   0.414474  0.396226        SVM   \n",
      "1  54.0   70.0  105.0  NEUTRAL   0.435484  0.339623        SVM   \n",
      "2  75.0   77.0   84.0      OBJ   0.493421  0.471698        SVM   \n",
      "3  93.0  115.0   66.0      POS   0.447115  0.584906        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.448113  0.443974  \n",
      "1         count_ng2  0.448113  0.443974  \n",
      "2         count_ng2  0.448113  0.443974  \n",
      "3         count_ng2  0.448113  0.443974  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   93.0   99.0      NEG   0.392157  0.377358        SVM   \n",
      "1  51.0   67.0  108.0  NEUTRAL   0.432203  0.320755        SVM   \n",
      "2  71.0   79.0   88.0      OBJ   0.473333  0.446541        SVM   \n",
      "3  94.0  121.0   65.0      POS   0.437209  0.591195        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.433962  0.428767  \n",
      "1         count_ng3  0.433962  0.428767  \n",
      "2         count_ng3  0.433962  0.428767  \n",
      "3         count_ng3  0.433962  0.428767  \n",
      "\n",
      "========================================\n",
      "4-balanced-Logistic Regression-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0  65.0   92.0   94.0      NEG   0.414013  0.408805  Logistic Regression   \n",
      "1  52.0   76.0  107.0  NEUTRAL   0.406250  0.327044  Logistic Regression   \n",
      "2  80.0   79.0   79.0      OBJ   0.503145  0.503145  Logistic Regression   \n",
      "3  91.0  101.0   68.0      POS   0.473958  0.572327  Logistic Regression   \n",
      "\n",
      "  feauter_generator      acc        f1  \n",
      "0         count_ng1  0.45283  0.448856  \n",
      "1         count_ng1  0.45283  0.448856  \n",
      "2         count_ng1  0.45283  0.448856  \n",
      "3         count_ng1  0.45283  0.448856  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   89.0   94.0      NEG   0.422078  0.408805        SVM   \n",
      "1  47.0   77.0  112.0  NEUTRAL   0.379032  0.295597        SVM   \n",
      "2  79.0   94.0   80.0      OBJ   0.456647  0.496855        SVM   \n",
      "3  82.0  103.0   77.0      POS   0.443243  0.515723        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.429245  0.425035  \n",
      "1         count_ng1  0.429245  0.425035  \n",
      "2         count_ng1  0.429245  0.425035  \n",
      "3         count_ng1  0.429245  0.425035  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-count_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  53.0   73.0  106.0      NEG   0.420635  0.333333  Perceptron   \n",
      "1  65.0  115.0   94.0  NEUTRAL   0.361111  0.408805  Perceptron   \n",
      "2  60.0   56.0   99.0      OBJ   0.517241  0.377358  Perceptron   \n",
      "3  95.0  119.0   64.0      POS   0.443925  0.597484  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.429245  0.425289  \n",
      "1         count_ng3  0.429245  0.425289  \n",
      "2         count_ng3  0.429245  0.425289  \n",
      "3         count_ng3  0.429245  0.425289  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  61.0   73.0   98.0      NEG   0.455224  0.383648  Perceptron   \n",
      "1  56.0   88.0  103.0  NEUTRAL   0.388889  0.352201  Perceptron   \n",
      "2  67.0   73.0   92.0      OBJ   0.478571  0.421384  Perceptron   \n",
      "3  94.0  124.0   65.0      POS   0.431193  0.591195  Perceptron   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.437107  0.433213  \n",
      "1         count_ng2  0.437107  0.433213  \n",
      "2         count_ng2  0.437107  0.433213  \n",
      "3         count_ng2  0.437107  0.433213  \n",
      "\n",
      "========================================\n",
      "4-balanced-Perceptron-count_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall  Classifier  \\\n",
      "0  60.0   56.0   99.0      NEG   0.517241  0.377358  Perceptron   \n",
      "1  76.0  145.0   83.0  NEUTRAL   0.343891  0.477987  Perceptron   \n",
      "2  84.0  110.0   75.0      OBJ   0.432990  0.528302  Perceptron   \n",
      "3  53.0   52.0  106.0      POS   0.504762  0.333333  Perceptron   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         count_ng1  0.429245  0.42845  \n",
      "1         count_ng1  0.429245  0.42845  \n",
      "2         count_ng1  0.429245  0.42845  \n",
      "3         count_ng1  0.429245  0.42845  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Passive Aggresive-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0    80.0   81.0  256.0      NEG   0.496894  0.238095  Passive Aggresive   \n",
      "1     8.0   24.0  158.0  NEUTRAL   0.250000  0.048193  Passive Aggresive   \n",
      "2  1263.0  502.0   75.0      OBJ   0.715581  0.943946  Passive Aggresive   \n",
      "3    17.0   24.0  142.0      POS   0.414634  0.106918  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.684342  0.619216  \n",
      "1         count_ng3  0.684342  0.619216  \n",
      "2         count_ng3  0.684342  0.619216  \n",
      "3         count_ng3  0.684342  0.619216  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Passive Aggresive-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0    88.0   85.0  248.0      NEG   0.508671  0.261905  Passive Aggresive   \n",
      "1    10.0   33.0  156.0  NEUTRAL   0.232558  0.060241  Passive Aggresive   \n",
      "2  1251.0  481.0   87.0      OBJ   0.722286  0.934978  Passive Aggresive   \n",
      "3    21.0   30.0  138.0      POS   0.411765  0.132075  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.685343  0.627471  \n",
      "1         count_ng2  0.685343  0.627471  \n",
      "2         count_ng2  0.685343  0.627471  \n",
      "3         count_ng2  0.685343  0.627471  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Passive Aggresive-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall         Classifier  \\\n",
      "0    86.0  142.0  250.0      NEG   0.377193  0.255952  Passive Aggresive   \n",
      "1    12.0   62.0  154.0  NEUTRAL   0.162162  0.072289  Passive Aggresive   \n",
      "2  1182.0  449.0  156.0      OBJ   0.724709  0.883408  Passive Aggresive   \n",
      "3    23.0   43.0  136.0      POS   0.348485  0.144654  Passive Aggresive   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng1  0.651826  0.608768  \n",
      "1         count_ng1  0.651826  0.608768  \n",
      "2         count_ng1  0.651826  0.608768  \n",
      "3         count_ng1  0.651826  0.608768  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-mnb-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0     0.0    2.0  336.0      NEG    0.00000  1.000000        mnb   \n",
      "1     1.0    0.0  165.0  NEUTRAL    1.00000  0.006024        mnb   \n",
      "2  1337.0  659.0    1.0      OBJ    0.66984  0.999253        mnb   \n",
      "3     0.0    0.0  159.0      POS    0.00000  0.000000        mnb   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.669335  0.537827  \n",
      "1         tfidf_ng3  0.669335  0.537827  \n",
      "2         tfidf_ng3  0.669335  0.537827  \n",
      "3         tfidf_ng3  0.669335  0.537827  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  65.0   89.0   94.0      NEG   0.422078  0.408805        sgd   \n",
      "1  55.0  100.0  104.0  NEUTRAL   0.354839  0.345912        sgd   \n",
      "2  87.0  103.0   72.0      OBJ   0.457895  0.547170        sgd   \n",
      "3  69.0   68.0   90.0      POS   0.503650  0.433962        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.433962  0.432609  \n",
      "1         tfidf_ng1  0.433962  0.432609  \n",
      "2         tfidf_ng1  0.433962  0.432609  \n",
      "3         tfidf_ng1  0.433962  0.432609  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  58.0   76.0  101.0      NEG   0.432836  0.364780        sgd   \n",
      "1  73.0  112.0   86.0  NEUTRAL   0.394595  0.459119        sgd   \n",
      "2  87.0   81.0   72.0      OBJ   0.517857  0.547170        sgd   \n",
      "3  80.0   69.0   79.0      POS   0.536913  0.503145        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng2  0.468553  0.467978  \n",
      "1         tfidf_ng2  0.468553  0.467978  \n",
      "2         tfidf_ng2  0.468553  0.467978  \n",
      "3         tfidf_ng2  0.468553  0.467978  \n",
      "\n",
      "========================================\n",
      "4-balanced-sgd-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  62.0   84.0  97.0      NEG   0.424658  0.389937        sgd   \n",
      "1  73.0  117.0  86.0  NEUTRAL   0.384211  0.459119        sgd   \n",
      "2  83.0   81.0  76.0      OBJ   0.506098  0.522013        sgd   \n",
      "3  72.0   64.0  87.0      POS   0.529412  0.452830        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.455975  0.456741  \n",
      "1         tfidf_ng3  0.455975  0.456741  \n",
      "2         tfidf_ng3  0.455975  0.456741  \n",
      "3         tfidf_ng3  0.455975  0.456741  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-sgd-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    93.0  110.0  243.0      NEG   0.458128  0.276786        sgd   \n",
      "1     9.0   24.0  157.0  NEUTRAL   0.272727  0.054217        sgd   \n",
      "2  1253.0  475.0   85.0      OBJ   0.725116  0.936472        sgd   \n",
      "3    20.0   15.0  139.0      POS   0.571429  0.125786        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.687844  0.628996  \n",
      "1         tfidf_ng3  0.687844  0.628996  \n",
      "2         tfidf_ng3  0.687844  0.628996  \n",
      "3         tfidf_ng3  0.687844  0.628996  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-sgd-tfidf_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    86.0  101.0  250.0      NEG   0.459893  0.255952        sgd   \n",
      "1     3.0   25.0  163.0  NEUTRAL   0.107143  0.018072        sgd   \n",
      "2  1255.0  494.0   83.0      OBJ   0.717553  0.937967        sgd   \n",
      "3    20.0   15.0  139.0      POS   0.571429  0.125786        sgd   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.682341  0.618474  \n",
      "1         tfidf_ng1  0.682341  0.618474  \n",
      "2         tfidf_ng1  0.682341  0.618474  \n",
      "3         tfidf_ng1  0.682341  0.618474  \n",
      "\n",
      "========================================\n",
      "4-balanced-KNN-count_ng2\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  76.0  212.0   83.0      NEG   0.263889  0.477987        KNN   \n",
      "1   5.0   12.0  154.0  NEUTRAL   0.294118  0.031447        KNN   \n",
      "2  53.0  102.0  106.0      OBJ   0.341935  0.333333        KNN   \n",
      "3  44.0  132.0  115.0      POS   0.250000  0.276730        KNN   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.279874  0.249282  \n",
      "1         count_ng2  0.279874  0.249282  \n",
      "2         count_ng2  0.279874  0.249282  \n",
      "3         count_ng2  0.279874  0.249282  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-SVM-tfidf_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0    97.0  110.0  239.0      NEG   0.468599  0.288690        SVM   \n",
      "1     9.0   32.0  157.0  NEUTRAL   0.219512  0.054217        SVM   \n",
      "2  1242.0  467.0   96.0      OBJ   0.726741  0.928251        SVM   \n",
      "3    23.0   19.0  136.0      POS   0.547619  0.144654        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.685843  0.631137  \n",
      "1         tfidf_ng3  0.685843  0.631137  \n",
      "2         tfidf_ng3  0.685843  0.631137  \n",
      "3         tfidf_ng3  0.685843  0.631137  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-tfidf_ng2\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  60.0   82.0  99.0      NEG   0.422535  0.377358        SVM   \n",
      "1  71.0  113.0  88.0  NEUTRAL   0.385870  0.446541        SVM   \n",
      "2  89.0   83.0  70.0      OBJ   0.517442  0.559748        SVM   \n",
      "3  75.0   63.0  84.0      POS   0.543478  0.471698        SVM   \n",
      "\n",
      "  feauter_generator       acc       f1  \n",
      "0         tfidf_ng2  0.463836  0.46387  \n",
      "1         tfidf_ng2  0.463836  0.46387  \n",
      "2         tfidf_ng2  0.463836  0.46387  \n",
      "3         tfidf_ng2  0.463836  0.46387  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-tfidf_ng3\n",
      "----------------------------------------\n",
      "     TP     FP     FN        L  Percision    Recall Classifier  \\\n",
      "0  53.0   75.0  106.0      NEG   0.414062  0.333333        SVM   \n",
      "1  74.0  123.0   85.0  NEUTRAL   0.375635  0.465409        SVM   \n",
      "2  88.0   91.0   71.0      OBJ   0.491620  0.553459        SVM   \n",
      "3  72.0   60.0   87.0      POS   0.545455  0.452830        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng3  0.451258  0.450156  \n",
      "1         tfidf_ng3  0.451258  0.450156  \n",
      "2         tfidf_ng3  0.451258  0.450156  \n",
      "3         tfidf_ng3  0.451258  0.450156  \n",
      "\n",
      "========================================\n",
      "4-balanced-SVM-tfidf_ng1\n",
      "----------------------------------------\n",
      "     TP     FP    FN        L  Percision    Recall Classifier  \\\n",
      "0  67.0   84.0  92.0      NEG   0.443709  0.421384        SVM   \n",
      "1  64.0  105.0  95.0  NEUTRAL   0.378698  0.402516        SVM   \n",
      "2  86.0   88.0  73.0      OBJ   0.494253  0.540881        SVM   \n",
      "3  75.0   67.0  84.0      POS   0.528169  0.471698        SVM   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         tfidf_ng1  0.459119  0.459339  \n",
      "1         tfidf_ng1  0.459119  0.459339  \n",
      "2         tfidf_ng1  0.459119  0.459339  \n",
      "3         tfidf_ng1  0.459119  0.459339  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Logistic Regression-count_ng1\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0    85.0  100.0  251.0      NEG   0.459459  0.252976  Logistic Regression   \n",
      "1     8.0   42.0  158.0  NEUTRAL   0.160000  0.048193  Logistic Regression   \n",
      "2  1252.0  480.0   86.0      OBJ   0.722864  0.935725  Logistic Regression   \n",
      "3    16.0   16.0  143.0      POS   0.500000  0.100629  Logistic Regression   \n",
      "\n",
      "  feauter_generator      acc        f1  \n",
      "0         count_ng1  0.68084  0.620255  \n",
      "1         count_ng1  0.68084  0.620255  \n",
      "2         count_ng1  0.68084  0.620255  \n",
      "3         count_ng1  0.68084  0.620255  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Logistic Regression-count_ng2\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0    80.0   67.0  256.0      NEG   0.544218  0.238095  Logistic Regression   \n",
      "1     7.0   21.0  159.0  NEUTRAL   0.250000  0.042169  Logistic Regression   \n",
      "2  1289.0  515.0   49.0      OBJ   0.714523  0.963378  Logistic Regression   \n",
      "3    10.0   10.0  149.0      POS   0.500000  0.062893  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng2  0.693347  0.619747  \n",
      "1         count_ng2  0.693347  0.619747  \n",
      "2         count_ng2  0.693347  0.619747  \n",
      "3         count_ng2  0.693347  0.619747  \n",
      "\n",
      "========================================\n",
      "4-unbalanced-Logistic Regression-count_ng3\n",
      "----------------------------------------\n",
      "       TP     FP     FN        L  Percision    Recall           Classifier  \\\n",
      "0    70.0   59.0  266.0      NEG   0.542636  0.208333  Logistic Regression   \n",
      "1     6.0   12.0  160.0  NEUTRAL   0.333333  0.036145  Logistic Regression   \n",
      "2  1297.0  537.0   41.0      OBJ   0.707197  0.969357  Logistic Regression   \n",
      "3    10.0    8.0  149.0      POS   0.555556  0.062893  Logistic Regression   \n",
      "\n",
      "  feauter_generator       acc        f1  \n",
      "0         count_ng3  0.691846  0.612378  \n",
      "1         count_ng3  0.691846  0.612378  \n",
      "2         count_ng3  0.691846  0.612378  \n",
      "3         count_ng3  0.691846  0.612378  \n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sun Apr 14 19:05:12 2013\n",
    "\n",
    "@author1: Mohamed Aly <mohamed@mohamedaly.info>\n",
    "@author2: Mahmoud Nabil <mah.nabil@yahoo.com>\n",
    "\n",
    "\n",
    "moved to jupyter notebook and edited by Rabab Alkhalifa <raalkhalifa@iau.edu.sa>\n",
    "\"\"\"\n",
    "\n",
    "# TODO: Initialize the three models\n",
    "clf_A = SVC(random_state=0)\n",
    "clf_B = DecisionTreeClassifier(random_state=0)\n",
    "clf_C = RandomForestClassifier( random_state=0)\n",
    "\n",
    "# TODO: Calculate the number of samples for 1%, 10%, and 100% of the training data\n",
    "# HINT: samples_100 is the entire training set i.e. len(y_train)\n",
    "# HINT: samples_10 is 10% of samples_100 (ensure to set the count of the values to be `int` and not `float`)\n",
    "# HINT: samples_1 is 1% of samples_100 (ensure to set the count of the values to be `int` and not `float`)\n",
    "\n",
    "\n",
    "\n",
    "gr = AraTweet()\n",
    "\n",
    "\n",
    "classifiers_collection = {}\n",
    "scores_collection = {}\n",
    "dataframe_collection = {}\n",
    "TweetCount_collection = {}\n",
    "results = {}\n",
    "\n",
    "FeatuerDataSet = []\n",
    "\n",
    "for data in datas:\n",
    "    scores = list()\n",
    "    \n",
    "    ###################################load the data####################################\n",
    "    print(\"\\n\" +\"=\"*40)\n",
    "    print(\"Loading data:\", data['name'])\n",
    "    print(\"\\n\" +\"=\"*40)\n",
    "    if (LoadValidation):\n",
    "        (d_train, y_train, d_test, y_test, d_valid, y_valid) = gr.get_train_test_validation(**data['params'])\n",
    "        if (Evaluate_On_TestSet):\n",
    "            d_train = np.concatenate((d_train, d_valid))\n",
    "            y_train = np.concatenate((y_train, y_valid))\n",
    "        else:\n",
    "            d_test = d_valid\n",
    "            y_test = y_valid\n",
    "    else:\n",
    "        (d_train, y_train, d_test, y_test) = gr.get_train_test(**data['params'])\n",
    "    \n",
    "    samples_100 = len(y_train)\n",
    "    samples_30 = int( len(y_train) * .30)\n",
    "    samples_10 = int( len(y_train) * .10)\n",
    "\n",
    "#     OBJ, NEG, NEUTRAL ,POS = GetCount(y_train) \n",
    "#     OBJ, NEG, NEUTRAL ,POS = GetCount(y_test) \n",
    "#     OBJ, NEG, NEUTRAL ,POS = GetCount(y_valid) \n",
    "    #---------------------------------------------------------------------------------\n",
    "    TweetCount = [GetCount(y_train ), GetCount(y_test), GetCount(y_valid) ]\n",
    "    df_tweetCount = pd.DataFrame(TweetCount,columns= ['OBJ', 'NEG', 'NEUTRAL' ,'POS']) \n",
    "    df_tweetCount['Dataset'] = ['Train Set','Test Set','Validation Set']\n",
    "    df_tweetCount['name'] = data['name']\n",
    "    TweetCount_collection[data['name']] = df_tweetCount\n",
    "    #---------------------------------------------------------------------------------\n",
    "    ####################################################################################\n",
    "    \n",
    "    for feat_generator in Features_Generators:\n",
    "        ####################################Features Generation#############################\n",
    "#         print(\"Features Generation:\", feat_generator['name'])\n",
    "        X_train = feat_generator['feat_generator'].fit_transform(d_train)\n",
    "        X_test = feat_generator['feat_generator'].transform(d_test)\n",
    "        ####################################################################################\n",
    "        \n",
    "        \n",
    "        FeatuerData = [data['name'], feat_generator['name'],X_train.shape[1]]\n",
    "        FeatuerDataSet.append(FeatuerData)\n",
    "        for clf in classifiers:\n",
    "            \n",
    "                    clf_name = clf[\"name\"]\n",
    "                    print(clf[\"name\"])\n",
    "                    results[clf_name] = {}\n",
    "                    for i, samples in enumerate([ samples_100]):\n",
    "                        results[clf_name][i] , pred , y_test = \\\n",
    "                        train_predict(clf['clf'], samples, X_train, y_train, X_test, y_test)\n",
    "\n",
    "                        (acc, tacc, support, f1 , df) = Evaluate_Result(pred, y_test)\n",
    "\n",
    "                        score = dict(data=data['name'],\n",
    "                                         feat_generator=feat_generator['name'],\n",
    "                                         clf=clf['name'],\n",
    "                                         # feat_ext=feat_ext['name'],\n",
    "                                         f1=f1,\n",
    "                                         acc=acc,\n",
    "                                         tacc=tacc)\n",
    "\n",
    "                        df['Classifier'] =  clf[\"name\"]\n",
    "                        df['feauter_generator'] = feat_generator['name']\n",
    "\n",
    "                        df['acc'] = acc\n",
    "                        df['f1']=f1\n",
    "\n",
    "\n",
    "\n",
    "                        dataframe_collection[ data['name'] + '-' + clf[\"name\"] + '-' + feat_generator['name'] ]  = df\n",
    "#                         print(clf[\"name\"],df['feauter_generator'] )\n",
    "                        scores.append(score)\n",
    "\n",
    "    for key in dataframe_collection.keys():\n",
    "        print(\"\\n\" +\"=\"*40)\n",
    "        print(key)\n",
    "        print(\"-\"*40)\n",
    "        print(dataframe_collection[key])\n",
    "\n",
    "        df_Classifiers = pd.DataFrame(classifiers)\n",
    "        classifiers_collection[data['name']] = df_Classifiers\n",
    "\n",
    "    #     print(\"-\"*40)\n",
    "    #     print(df_Classifiers.head())\n",
    "\n",
    "        df_s = pd.DataFrame(scores)\n",
    "        scores_collection[data['name']] = df_s\n",
    "    #     print(\"-\"*40)\n",
    "    #     print(df_s.head())\n",
    "df_feauterCount = pd.DataFrame(FeatuerDataSet, columns=['DataSet_Name','Feauter_Generator','Feauters+Count'])\n",
    "    \n",
    "# ####################################Testing##############################################\n",
    "# print(60 * \"=\")\n",
    "# for s in scores:\n",
    "#     print(\"\")\n",
    "#     for k, v in s.iteritems():\n",
    "#         print(k, v)-"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>clf</th>\n",
       "      <th>data</th>\n",
       "      <th>f1</th>\n",
       "      <th>feat_generator</th>\n",
       "      <th>tacc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.693347</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>4-unbalanced</td>\n",
       "      <td>0.619747</td>\n",
       "      <td>count_ng2</td>\n",
       "      <td>0.693347</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         acc                  clf          data        f1 feat_generator  \\\n",
       "10  0.693347  Logistic Regression  4-unbalanced  0.619747      count_ng2   \n",
       "\n",
       "        tacc  \n",
       "10  0.693347  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balanced_scores = scores_collection['4-unbalanced']\n",
    "subsetDataFrame = balanced_scores #[balanced_scores['feat_generator'].isin(['count_ng1', 'count_ng2','count_ng3'])]\n",
    "subsetDataFrame[subsetDataFrame['acc'] > .693]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>clf</th>\n",
       "      <th>data</th>\n",
       "      <th>f1</th>\n",
       "      <th>feat_generator</th>\n",
       "      <th>tacc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.490566</td>\n",
       "      <td>mnb</td>\n",
       "      <td>4-balanced</td>\n",
       "      <td>0.491652</td>\n",
       "      <td>tfidf_ng2</td>\n",
       "      <td>0.490566</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         acc  clf        data        f1 feat_generator      tacc\n",
       "49  0.490566  mnb  4-balanced  0.491652      tfidf_ng2  0.490566"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balanced_scores = scores_collection['4-balanced']\n",
    "subsetDataFrame = balanced_scores #[balanced_scores['feat_generator'].isin(['count_ng1', 'count_ng2','count_ng3'])]\n",
    "subsetDataFrame[subsetDataFrame['acc'] > .490]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n",
      "C:\\Users\\Home\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:756: DeprecationWarning: The default `weighted` averaging is deprecated, and from version 0.18, use of precision, recall or F-score with multiclass or multilabel data or pos_label=None will result in an exception. Please set an explicit value for `average`, one of (None, 'micro', 'macro', 'weighted', 'samples'). In cross validation use, for instance, scoring=\"f1_weighted\" instead of scoring=\"f1\".\n",
      "  sample_weight=sample_weight)\n"
     ]
    }
   ],
   "source": [
    "# TODO: Import 'GridSearchCV', 'make_scorer', and any other necessary libraries\n",
    "\n",
    "\n",
    "\n",
    "# TODO: Initialize the classifier\n",
    "clf = RandomForestClassifier( random_state=0)\n",
    "\n",
    "# TODO: Create the parameters list you wish to tune, using a dictionary if needed.\n",
    "# HINT: parameters = {'parameter_1': [value1, value2], 'parameter_2': [value1, value2]}\n",
    "\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10 ,20]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4 , 8]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "# Create the random grid\n",
    "parameters = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf\n",
    "              ,'bootstrap': bootstrap}\n",
    "\n",
    "#Ref. https://towardsdatascience.com/hyperparameter-tuning-the-random-forest-in-python-using-scikit-learn-28d2aa77dd74\n",
    "\n",
    "# TODO: Make an fbeta_score scoring object using make_scorer()\n",
    "scorer = make_scorer(fbeta_score, beta=0.5)\n",
    "\n",
    "# TODO: Perform grid search on the classifier using 'scorer' as the scoring method using GridSearchCV()\n",
    "grid_obj =  GridSearchCV(clf , parameters ,scoring=scorer)\n",
    "\n",
    "# TODO: Fit the grid search object to the training data and find the optimal parameters using fit()\n",
    "grid_fit = grid_obj.fit(X_train,y_train)\n",
    "\n",
    "# Get the estimator\n",
    "best_clf = grid_fit.best_estimator_\n",
    "\n",
    "print(best_clf)\n",
    "\n",
    "# Make predictions using the unoptimized and model\n",
    "predictions = (clf.fit(X_train, y_train)).predict(X_test)\n",
    "best_predictions = best_clf.predict(X_test)\n",
    "\n",
    "# Report the before-and-afterscores\n",
    "print(\"Unoptimized model\\n------\")\n",
    "print(\"Accuracy score on testing data: {:.4f}\".format(accuracy_score(y_test, predictions)))\n",
    "print(\"F-score on testing data: {:.4f}\".format(fbeta_score(y_test, predictions, beta = 0.5,average='macro')))\n",
    "print(\"\\nOptimized Model\\n------\")\n",
    "print(\"Final accuracy score on the testing data: {:.4f}\".format(accuracy_score(y_test, best_predictions)))\n",
    "print(\"Final F-score on the testing data: {:.4f}\".format(fbeta_score(y_test, best_predictions, beta = 0.5,average='macro')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step _: Fine tune the standard model ###  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "###### -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sun Apr 14 19:05:12 2013\n",
    "\n",
    "@author1: Mohamed Aly <mohamed@mohamedaly.info>\n",
    "@author2: Mahmoud Nabil <mah.nabil@yahoo.com>\n",
    "\n",
    "\n",
    "moved to jupyter notebook and edited by Rabab Alkhalifa <raalkhalifa@iau.edu.sa>\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "gr = AraTweet()\n",
    "\n",
    "\n",
    "classifiers_collection = {}\n",
    "scores_collection = {}\n",
    "dataframe_collection = {}\n",
    "TweetCount_collection = {}\n",
    "\n",
    "\n",
    "FeatuerDataSet = []\n",
    "for data in datas:\n",
    "    scores = list()\n",
    "    \n",
    "    ###################################load the data####################################\n",
    "    print(\"\\n\" +\"=\"*40)\n",
    "    print(\"Loading data:\", data['name'])\n",
    "    print(\"\\n\" +\"=\"*40)\n",
    "    if (LoadValidation):\n",
    "        (d_train, y_train, d_test, y_test, d_valid, y_valid) = gr.get_train_test_validation(**data['params'])\n",
    "        if (Evaluate_On_TestSet):\n",
    "            d_train = np.concatenate((d_train, d_valid))\n",
    "            y_train = np.concatenate((y_train, y_valid))\n",
    "        else:\n",
    "            d_test = d_valid\n",
    "            y_test = y_valid\n",
    "    else:\n",
    "        (d_train, y_train, d_test, y_test) = gr.get_train_test(**data['params'])\n",
    "    \n",
    "\n",
    "#     OBJ, NEG, NEUTRAL ,POS = GetCount(y_train) \n",
    "#     OBJ, NEG, NEUTRAL ,POS = GetCount(y_test) \n",
    "#     OBJ, NEG, NEUTRAL ,POS = GetCount(y_valid) \n",
    "    #---------------------------------------------------------------------------------\n",
    "    TweetCount = [GetCount(y_train ), GetCount(y_test), GetCount(y_valid) ]\n",
    "    df_tweetCount = pd.DataFrame(TweetCount,columns= ['OBJ', 'NEG', 'NEUTRAL' ,'POS']) \n",
    "    df_tweetCount['Dataset'] = ['Train Set','Test Set','Validation Set']\n",
    "    df_tweetCount['name'] = data['name']\n",
    "    TweetCount_collection[data['name']] = df_tweetCount\n",
    "    #---------------------------------------------------------------------------------\n",
    "    ####################################################################################\n",
    "    \n",
    "    for feat_generator in Features_Generators:\n",
    "        ####################################Features Generation#############################\n",
    "#         print(\"Features Generation:\", feat_generator['name'])\n",
    "        X_train = feat_generator['feat_generator'].fit_transform(d_train)\n",
    "        X_test = feat_generator['feat_generator'].transform(d_test)\n",
    "        ####################################################################################\n",
    "        \n",
    "        \n",
    "        FeatuerData = [data['name'], feat_generator['name'],X_train.shape[1]]\n",
    "        FeatuerDataSet.append(FeatuerData)\n",
    "        for clf in classifiers:\n",
    "                    \n",
    "                    if clf['parameter_tunning']:\n",
    "                        print(\"\\n------\\n\")\n",
    "                        print(\"tuning: \", data['name'] , clf[\"name\"] ,feat_generator['name'])\n",
    "                        print(\"\\n------\\n\")\n",
    "                        \n",
    "                        grid_fit = clf['tune_clf'].fit(X_train, y_train)\n",
    "                        best_clf = grid_fit.best_estimator_\n",
    "                        \n",
    "                        # Make predictions using the unoptimized and model\n",
    "                        predictions = (clf[\"clf\"].fit(X_train, y_train)).predict(X_test)\n",
    "                        best_predictions = best_clf.predict(X_test)\n",
    "\n",
    "                        accs = accuracy_score(y_test, best_predictions)\n",
    "                        f1_scores = f1_score(y_test, best_predictions,average='macro')\n",
    "                        parameter_settings = grid_fit.best_params_\n",
    "                        \n",
    "                        entries.append(data['name'] , clf[\"name\"] ,feat_generator['name'],accs,f1_scores,parameter_settings,best_clf)\n",
    "                        \n",
    "                        \n",
    "#                         print(\"Final F-score on the testing data: {:.3f}\".format(fbeta_score(y_test, best_predictions, beta = 0.5,average='macro')))\n",
    "#                         # Report the before-and-afterscores\n",
    "#                         print(\"Best Settings\", grid_fit.best_score_ , (grid_fit.best_params_))\n",
    "#                         print(\"\\n Unoptimized model\\n------\")\n",
    "#                         print(\"Accuracy score on testing data: {:.3f}\".format(accuracy_score(y_test, predictions)))\n",
    "#                         print(\"F-score on testing data: {:.3f}\".format(fbeta_score(y_test, predictions, beta = 0.5,average='macro')))\n",
    "#                         print(\"\\nOptimized Model\\n------\")\n",
    "#                         print(\"Final accuracy score on the testing data: {:.3f}\".format(accuracy_score(y_test, best_predictions)))\n",
    "#                         print(\"Final F-score on the testing data: {:.3f}\".format(fbeta_score(y_test, best_predictions, beta = 0.5,average='macro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv_df = pd.DataFrame(entries, columns=['data','model_name','feauter_generator','accuracy','f1_Score','parameter_settings','best_CLF'])\n",
    "subset = cv_df[cv_df['data'] == '4-unbalanced']\n",
    "\n",
    "subset[subset['accuracy'] > .694]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset = cv_df[cv_df['data'] == '4-balanced']\n",
    "\n",
    "subset[subset['accuracy'] > .491]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### using cross validation with different folds to train the data with the default parameter setting of the classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "========================================\n",
      "('Loading data:', '4-balanced')\n",
      "\n",
      "========================================\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-balanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "========================================\n",
      "('Loading data:', '4-unbalanced')\n",
      "\n",
      "========================================\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Logistic Regression')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Passive Aggresive')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'SVM')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'Perceptron')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'bnb')\n",
      "\n",
      "------\n",
      "\n",
      "\n",
      "------\n",
      "\n",
      "('tuning: ', '4-unbalanced', 'mnb')\n",
      "\n",
      "------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "###### -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sun Apr 14 19:05:12 2013\n",
    "\n",
    "@author1: Mohamed Aly <mohamed@mohamedaly.info>\n",
    "@author2: Mahmoud Nabil <mah.nabil@yahoo.com>\n",
    "\n",
    "\n",
    "moved to jupyter notebook and edited by Rabab Alkhalifa <raalkhalifa@iau.edu.sa>\n",
    "\"\"\"\n",
    "\n",
    "CV = 5\n",
    "gr = AraTweet()\n",
    "entries = []\n",
    "cv_df = pd.DataFrame(index=range(CV * len(classifiers)))\n",
    "for data in datas:\n",
    "    scores = list()\n",
    "    \n",
    "    ###################################load the data####################################\n",
    "    print(\"\\n\" +\"=\"*40)\n",
    "    print(\"Loading data:\", data['name'])\n",
    "    print(\"\\n\" +\"=\"*40)\n",
    "    if (LoadValidation):\n",
    "        (d_train, y_train, d_test, y_test, d_valid, y_valid) = gr.get_train_test_validation(**data['params'])\n",
    "        if (Evaluate_On_TestSet):\n",
    "            d_train = np.concatenate((d_train, d_valid))\n",
    "            y_train = np.concatenate((y_train, y_valid))\n",
    "        else:\n",
    "            d_test = d_valid\n",
    "            y_test = y_valid\n",
    "    else:\n",
    "        (d_train, y_train, d_test, y_test) = gr.get_train_test(**data['params'])\n",
    "    \n",
    "     \n",
    "    ####################################################################################\n",
    "    \n",
    "    for feat_generator in Features_Generators:\n",
    "        ####################################Features Generation#############################\n",
    "#         print(\"Features Generation:\", feat_generator['name'])\n",
    "        X_train = feat_generator['feat_generator'].fit_transform(d_train)\n",
    "        X_test = feat_generator['feat_generator'].transform(d_test)\n",
    "        ####################################################################################\n",
    "        \n",
    "        tweets = sp.vstack((X_train , X_test))\n",
    "        classes = np.concatenate((y_train , y_test))\n",
    "        \n",
    "       \n",
    "        for clf in classifiers:\n",
    "                    \n",
    "                    if clf['parameter_tunning']:\n",
    "                        print(\"\\n------\\n\")\n",
    "                        print(\"tuning: \", data['name'] , clf[\"name\"] )\n",
    "                        print(\"\\n------\\n\")\n",
    "                        \n",
    "                        \n",
    "                        model_name = clf['clf'].__class__.__name__\n",
    "                        accuracies = cross_val_score(clf['clf'], tweets, classes, scoring='accuracy', cv=CV)\n",
    "                        for fold_idx, accuracy in enumerate(accuracies):\n",
    "                            entries.append((model_name,data['name'],feat_generator['name'], fold_idx, accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>data</th>\n",
       "      <th>feauter_generator</th>\n",
       "      <th>fold_idx</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>4-unbalanced</td>\n",
       "      <td>count_ng1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.696152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>4-unbalanced</td>\n",
       "      <td>count_ng2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.697802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>4-unbalanced</td>\n",
       "      <td>count_ng3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.694306</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             model_name          data feauter_generator  fold_idx  accuracy\n",
       "182  LogisticRegression  4-unbalanced         count_ng1         2  0.696152\n",
       "211  LogisticRegression  4-unbalanced         count_ng2         1  0.697802\n",
       "241  LogisticRegression  4-unbalanced         count_ng3         1  0.694306"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_df = pd.DataFrame(entries, columns=['model_name','data','feauter_generator' , 'fold_idx', 'accuracy'])\n",
    "subset = cv_df[cv_df['data'] == '4-unbalanced']\n",
    "\n",
    "subset[subset['accuracy'] > .694]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>data</th>\n",
       "      <th>feauter_generator</th>\n",
       "      <th>fold_idx</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>MultinomialNB</td>\n",
       "      <td>4-balanced</td>\n",
       "      <td>tfidf_ng2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.490566</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        model_name        data feauter_generator  fold_idx  accuracy\n",
       "149  MultinomialNB  4-balanced         tfidf_ng2         4  0.490566"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_df = pd.DataFrame(entries, columns=['model_name','data','feauter_generator' , 'fold_idx', 'accuracy'])\n",
    "subset = cv_df[cv_df['data'] == '4-balanced']\n",
    "\n",
    "subset[subset['accuracy'] > .49]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step _: Performance Measures for Classifieres ###  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metrics and the Naive Predictor\n",
    "*CharityML*, equipped with their research, knows individuals that make more than \\$50,000 are most likely to donate to their charity. Because of this, *CharityML* is particularly interested in predicting who makes more than \\$50,000 accurately. It would seem that using **accuracy** as a metric for evaluating a particular model's performace would be appropriate. Additionally, identifying someone that *does not* make more than \\$50,000 as someone who does would be detrimental to *CharityML*, since they are looking to find individuals willing to donate. Therefore, a model's ability to precisely predict those that make more than \\$50,000 is *more important* than the model's ability to **recall** those individuals. We can use **F-beta score** as a metric that considers both precision and recall:\n",
    "\n",
    "$$ F_{\\beta} = (1 + \\beta^2) \\cdot \\frac{precision \\cdot recall}{\\left( \\beta^2 \\cdot precision \\right) + recall} $$\n",
    "\n",
    "In particular, when $\\beta = 0.5$, more emphasis is placed on precision. This is called the **F$_{0.5}$ score** (or F-score for simplicity).\n",
    "\n",
    "Looking at the distribution of classes (those who make at most \\$50,000, and those who make more), it's clear most individuals do not make more than \\$50,000. This can greatly affect **accuracy**, since we could simply say *\"this person does not make more than \\$50,000\"* and generally be right, without ever looking at the data! Making such a statement would be called **naive**, since we have not considered any information to substantiate the claim. It is always important to consider the *naive prediction* for your data, to help establish a benchmark for whether a model is performing well. That been said, using that prediction would be pointless: If we predicted all people made less than \\$50,000, *CharityML* would identify no one as donors. \n",
    "\n",
    "\n",
    "#### Note: Recap of accuracy, precision, recall\n",
    "\n",
    "** Accuracy ** measures how often the classifier makes the correct prediction. It’s the ratio of the number of correct predictions to the total number of predictions (the number of test data points).\n",
    "\n",
    "** Precision ** tells us what proportion of messages we classified as spam, actually were spam.\n",
    "It is a ratio of true positives(words classified as spam, and which are actually spam) to all positives(all words classified as spam, irrespective of whether that was the correct classificatio), in other words it is the ratio of\n",
    "\n",
    "`[True Positives/(True Positives + False Positives)]`\n",
    "\n",
    "** Recall(sensitivity)** tells us what proportion of messages that actually were spam were classified by us as spam.\n",
    "It is a ratio of true positives(words classified as spam, and which are actually spam) to all the words that were actually spam, in other words it is the ratio of\n",
    "\n",
    "`[True Positives/(True Positives + False Negatives)]`\n",
    "\n",
    "For classification problems that are skewed in their classification distributions like in our case, for example if we had a 100 text messages and only 2 were spam and the rest 98 weren't, accuracy by itself is not a very good metric. We could classify 90 messages as not spam(including the 2 that were spam but we classify them as not spam, hence they would be false negatives) and 10 as spam(all 10 false positives) and still get a reasonably good accuracy score. For such cases, precision and recall come in very handy. These two metrics can be combined to get the F1 score, which is weighted average(harmonic mean) of the precision and recall scores. This score can range from 0 to 1, with 1 being the best possible F1 score(we take the harmonic mean as we are dealing with ratios).\n",
    "\n",
    "\n",
    "Ref: Udacity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "# print('Accuracy score: ', format(accuracy_score(rating, predictions)))\n",
    "# print('Precision score: ', format(precision_score(rating, predictions)))\n",
    "# print('Recall score: ', format(recall_score(rating, predictions)))\n",
    "# print('F1 score: ', format(f1_score(rating, predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 7: Conclusion ###"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {
   "attach-environment": true,
   "summary": "This is my model ",
   "thumbnail": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAYAAAAEgCAYAAAC926RRAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAABQtSURBVHhe7d0NsFTlfcfx3aJoiqm8dMMwgLlWKY42o7IMkJemrfIa2kJbM9VJA50hpTOQVKPTcp1JSgI2veRFK0nDFMQUpk40Y2whQiQMMbGmAdmrSCLGgogFysuNvCRNUy16+/ud82xm2XIvZ2HZvXef72fmO+c5Z8HJ3An3v3vO2d0cAAAAAAAAAAAAAAAAAAAAWtVYtaOin6g71FC1We0O2yHK8mq52qN2qnGqbK7yn3deAwD6iQHqsHqn+qxqV+btsnSZ+4D6pvIgmKS2KfPA2Bu2HhZel4cGAKCPm6q+ly5zL6kR6TLZet/+Qd2WLhPlP+djfqys+s8BABrIz9Jr8aB6Vn1JnVCDlfm/c1x5/3HVoZ5WtkUtUr+tLlX3KPuk+rn6fLJ3BsOGDetua2sLewCALDo7O3+sTSHd61ktA2Cg+k91nTqiKgeAeQD4lM75DoD5odwVV1xRfPXVV70EAGSUz+c7tRmf7vXsl8I2ixnKz/79y9+8rTwFdDRd5g6q0ekyMUr5WE/Hq61U/h8+vlA46wADAJyjWgaAz9d/NV0m1qvynTzerkuXyfE5qnwR+KQ6pDYpX0PwqwTntY8BAJog6wAYpKaox5K9lE/z+Jhv6Zwc9m2j8h0+vg10lVqg7JhaqraHloRjAIAmqPUicEMVi8XuUqkU9gAAWVyIawAAgBbCAACASDEAACBSDAAAiFR0F4Hb2jeEVd+2r2NmWAFAbbgIDADoFQMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiFTWATBYPap+pF5U71ZD1Wa1O2yHKPO3jC1Xe9RONU6VzVX+885rAECTZB0A96sn1DXqeuUh0K62qDFh632boXzMzVcrlHlgLFYT1YSwLg8NAECDZRkAl6v3q9XJXi73hjqhZqk1PhC2s9Nlcnyt6lZblV89jFDTlF8pHFPHw3q6AgA0QZYBcKXqUl9Rz6kH1CA1XB1Sdlh530aq/ekycUD5WE/HAQBNkGUAXKR8Ht+ncm5UP1Pl0z1lfrbv6sGnjUquq8tzBwBwIWQZAH6m7rYle+nFYA+EI8qndszbo+kyd1CNTpeJUcrHejpebaUa7wqFQnIAAFB/WQaAT+/41M3YZC+Xu1ntUutV+U4eb9ely+T4HOW7gSapk8qnijapqcoXfp3XPgYAaIIsA8A+ph5Svq3zBvUZ1aGmKN/SOTns20a1V/k20FVqgTJf/F2qtoeWhGMAgCbws/Q+q1gsdpdKvhxQP23tG8Kqb9vXMTOsAKA2+Xy+UxufSu9V1lcAAIAWwwAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACLFAACASGUdAPvUD9QOVfIBGao2q91hO0RZXi1Xe9RONU6VzVX+885rAECT1PIK4HfUDWp8spfLtastakzYet9mKB9z89UKZR4Yi9VENSGsy0MDANBg53MKaJZaky6T7ex0mRxfq7rVVjVYjVDTlF8pHFPHw3q6AgA0QdYB4F/m31Kdys/qbbg6lC5zh5X3baTany4TB5SP9XQcANAEWQfA+5TP5fv0zkL1flXJA8LVgweMrzOUurq6kgMAgPrLOgAOhu1R9c/K5/CPKJ/aMW/9mPnPjk6XiVHKx3o6Xm2l8nWG8YVCITkAAKi/LANgkHp7ukzWU9UP1XpVvpPH23XpMjk+R/luoEnqpPKpok3Kf9cXfp3XPgYAaIIsA8Dn9p9Wz6tn1Ab1hOpQU5Rv6Zwc9m2j2qt8G+gqtUCZL/4uVdtDS8IxAEAT+Fl6n1UsFrtLpfLbDuqjrd3zq+/b1zEzrACgNvl83jfslG/Z71HWawAAgBbDAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBI1TIABqjn1OPJXi53pdqm9qhH1EBllyjv+7gfb1Nldysff0lN8wEAQHPUMgBuVy+my8QydZ+6Wh1X85R5630f9+P+c3atulVdp6arLysPFQBAE2QdAKPUTPVAspfL5dVN6tFkL5dbo2any9ws5X3z4zcr/3kff1i9rl5RfiUwQQEAmiDrAPg79VfqrWQvlxumTqhTyV4ud0CNTJfJdn+6TB4/qfznK49b5d8BADRYlgHwu+qo6kz2Lrz5quS6urqSAwCA+ssyAN6rfl/tUz6F41M/96vB6iJlPkV0MF0m29HpMnn8cvWaqjxulX+n0ko13hUKheQAAKD+sgwA37njX9a+m8cXcb+tPqSeVLcom6vWpcvceuV98+P+893Kx/33fZeQ7yAao55RAIAmyHoN4EwWqTuVL+b6HP9qZd5638f9eLuyF9TX1C71hFqo3lQAgCaodQB8R/magO1VvovHt3t+UPnuHvsf5X0f9+P+c2V/o65SY9U3fQAA0Bzn8woAANCPMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUlkGwKXqGfW8ekF9WtmVapvaox5RA5Vdorzv4368TZXdrXz8JTXNBwAAzZFlALyublLXqxvUdDVJLVP3qavVcTVPmbfe93E/7j9n16pb1XXK/40vqwEKANAEWQZAt/qvdJm7OORjHgqPKlujZqfL3CzlffPjN6u88vGHlQfKK8qvBCYoAEATZL0G4GfqO9RRtVm9rE6oU8oOqJHpMtnuT5fJ4yfVMFV53Cr/DgCgwbIOgDeVT/+MUn7Wfo26UOarkuvq6koOAADqL+sAKPOz/ifVu9VgdZEyD4aD6TLZjk6XyeOXq9dU5XGr/DuVVqrxrlAoJAcAAPWXZQD4t7B/2dvb1BT1ovIguEXZXLUuXebWK++bH/+28jUDH/dFYN8l5DuIxijfXQQAaIIsA2CE8i/7nWq78jWAx9UidafyxVyf41+tzFvv+7gfb1fmW0i/pnapJ9RC5VNLAIAm8N05fVaxWOwulXw5oH7a2jeEVd+2r2NmWAFAbfL5fKc2PpXeq1qvAQAAWgQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACKVZQCMVk+qXeoFdbuyoWqz2h22Q5Tl1XK1R+1U41TZXOU/77wGADRJlgFwSt2lrlWT1MKwbldb1Jiw9b7NUD7m5qsVyjwwFquJakJYl4cGAKDBsgyAQ+rZdJn7qXpRjVSz1Bpl3s5Ol8nxtapbbVWD1Qg1TfmVwjF1PKynKwBAE9R6DaBN3ai2qeHKw8EOK++bh8P+dJk4oHysp+MAgCaoZQBcpr6u7lA/8YEKfrbv6sGnjUquq6srOQAAqL+sA+Bi5V/+D6nHfECOKJ/aMW+PpsvcQeULx2WjlI/1dLzaSjXeFQqF5AAAoP6yDADf1bNa+dz/vT4QrFflO3m8XZcuk+NzlP+eLxqfVD5VtElNVb7w67z2MQBAE2QZAO9VH1Y3qR2hD6gONUX5ls7JYd82qr3Kt4GuUguU+eLvUrU9tCQcAwA0gZ+l91nFYrG7VPLlgPppa98QVn3bvo6ZYQUAtcnn853a+FR6r2q9CwgA0CIYAAAQKQYAAESKAQAAkeIicB/VXy4C8/ME+h4uAgMAesUAAIBIMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACKVZQA8qI6qHyZ7qaFqs9odtkOU+RvGlqs9aqcap8rmKv955zUAoImyDIB/VNPT5S+0qy1qTNh632YoH3Pz1QplHhiL1UQ1IazLQwMA0ARZBsBT6li6/IVZak26TLaz02VyfK3qVlvVYDVCTVN+peD/zvGwrh4qAIAGOtdrAMPVoXSZO6y8byPV/nSZOKB8rKfjZ+JXDv4m+FJXV1dyAABQf/W4COxn+65eVip/m/34QqGQHAAA1N+5DoAjyqd2zFtfJLaDanS6TIxSPtbTcQBAk5zrAFivynfyeLsuXSbH5yjfDTRJnVQ+VbRJTVW+8Ou89jEAQJNkGQBfVd9XY5XP3c9THWqK8i2dk8O+bVR7lW8DXaUWKPPF36Vqe2hJOAYAaJIsA+A25dM8FyufulmtXlM3K9/u6QFQ/mXuawEL1VXqXcoXc8v8foKrQ1/xAQBA8/hUTZ9VLBa7S6XKGXL+2to3hFXftq9jZlj1bfw864ufJ+ohn893auObaXrFAOij+IVVX/w866s//Dxj/llmHQD1uA0UANAPMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAixQAAgEgxAAAgUgwAAIgUAwAAIsUAAIBIMQAAIFIMAACIFAMAACLFAACASDEAACBSDAAAiBQDAAAi1YwBMF29pPaodh8AADReowfAAPX3aoa6Vt0WtgCABmv0AJig/Mx/r3pDPaxmKQBAg+XDtlFuUT4F9JFkL5f7sJqoPprspeaHbKzy6aK+7lfVj9Ml6oCfZ33x86yf/vKzfKcqpMu+wwPggXSZ8AD4Urrs10phi/rg51lf/Dzrp6V+lo0+BXRQjU6XiVHKxwAADdboAbBdjVFXqoHqVrVeAQAazHflNNJbard6SH1M/ZP6umoFnWGL+uDnWV/8POuHnyUAAAAAAAAAAAAAAACa54/CFtn8sro4XSb8yQQfV3+Y7LWARn8URCv4oupOl//P6+pl5dtcf+oDOKvr1FWq/H6Q+9Tl6TJ5l/iz6RJ18B/qinSJDJ5S85RvXb9aPaP8b9sfYOn13apfYwDUbm7YnslFyr/Q3qWm+ADO6hvqb9W/JXu53C71SeVnX37GOluhPvarynfio3c/UP63bEvVULVQ+U2sfi9A+THgNBvDFmdX/dkqW8PWng5b1IdfASC7nWFr31OVT0aeD9t+jVcAtfOnAfpZwHH1oPqc+k3lUz93KX/cNbLzp7363OqZ/Lv69XSJjPys9UynKP1v3T/LS5I9ZOFPKjis/Hll/vIqf4TNf6vB6rvqeoXIfEt9RvlagE9X/KW6Rv2Z+o5CbZ5U/kjwapMUP8/a+WOAewvZvU35F//9qvKX/XuUP8kYESq/9PMzquqX1DvCFtn5S4JeUYvV74U+FY75MdTH+5S/jQ+1u1T9RsjrltGM7wTu794MW7/Mrv5iCH/YHWrjuyn8CsAfTPinIf//0q8A/BjO3Y3Kpyj3KV/E/JFCdr6p47PKF8/XqLVh7WOVt4ciIieUb1n03SvldXnf1wVQm18J2zPhlsXa+Ty/X035l70vovtTd19VqJ1vSfYXWL092Uv5/68rlU8L9XtcBK7db4Wtzw/6uw38SsAXfn+uzBeHkJ3v8x+XLnNb1M3pMlH5GLLxq9B/Vb5/vXxDgr+D+9fSJWrg+/89UKsvqvvVqges//0jMn7p55eAPv3jX1CuS/mlNi8La/dc2Frl2qr3cXa+VfFh5VMVq5QHqq+noHa+C60nvT3Wb3ANoHb+5T9E+ZYwPzt1fier3736eYXaVD67qn6mVb2Ps/sX5W/a851pvsPqDvUOtUJNVcjOd/nNSZen+RPVEtdTOAVUO14W1tcBda/y/xf9OStem/f9y4t3rp4/P2H5oPpjVXmKDb0bqR5TPr1b/haw8cqnf/9A9fvvM2cA1K63NyfxxqXa+YJlbz4dtjg/fvPSR9U9yR5qcZPyR7yYXxX4WhUi5ZfYPb0s5Avu0Wx+xeS7VB5XH1H+TKUvqKOqJe5caSDf8+9Xof5Qwj9Xvi20pfAKoHYt/7Kwwf46bM/Ep9l8/zqy83l/34n2fTVd+ZTPC8qn1/yxBsjuEfW/yndVzVB+P4UHQstgAJw7XhbWhz8/qdog5dsYh6nLfACZ+Z3qlR9bcET5/RT+qHLUpvLTQP3s329M5LZk4ALxG24+oXzb4jLlu1dQGw8AX/T1Rxe76n1kV/1dFNX7/R6vANAX+BfTnepDym+597lq3lV9bnyawm8GO9O/bZ9S4w1h2fljX36WLpOfp0/z+tNAvfbPsrd3sQPIwG+g80dpL1Kc7jl/fOInMuMVAJrNz1Z9fvqUqnxvBc+yzg0fnwEAkeLjM5AZrwCA1uL7/f1ZQD35i7AFGABAi/FHP/f23gpfZAcAtKCWu1URFw6fBgq0ljfCFjgrTgEBraWoKu+m8trfXeHvBwBOwwAAWos/C6ia32g3UN2mdvgAACAe/sDCp9IlACA2XCDGabgIDMRhuKq8NgBwDQBoMV9U1b/ofQ3gPep29Q0fAIwBALSWuWFb5mHwmtqu/C5hAECL6u3D8/zFMACAFlV5obf6W+q4CIzTcBEYaC2Vp3WrvwGMU744DQMAaC3V7wKuxF1AOA3PCIDWckDdq/xv++Nhbd6/Q41O9gAZELYAWoO/WP8S5Y9+8Dl/ryv3v6sAAEDMOAUEtJbevgzG1wCWpkuAAQC0mrvCttIgNU8NU5f5AACgtfl6wCfUK2qZeocCALQw3/9/j/Iv/k+pIQoA0OI+p15WixSne9ArrgEAreUt9bo6pSrf+OV/697v7bOCAAAAAAAAAAAAAABA/5TL/R8sv5OcKAHD8wAAAABJRU5ErkJggg=="
  },
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}