features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from nltk.tokenize import word_tokenize\n",
    "import string\n",
    "from nltk import pos_tag, ne_chunk\n",
    "from gensim import corpora\n",
    "from gensim.models import LdaModel\n",
    "import textstat\n",
    "from language_tool_python import LanguageTool\n",
    "import joblib\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('DATA_SET/final_test_v2.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic NLP Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['char_count'] = df['cleaned_text'].apply(len)\n",
    "\n",
    "df['word_count'] = df['cleaned_text'].apply(lambda x: len(word_tokenize(x))) # word count\n",
    "\n",
    "df['word_density'] = df['word_count'] / df['char_count'] # word density\n",
    "\n",
    "# punctuation count\n",
    "def punctuation_count(text):\n",
    "    return sum(1 for char in text if char in string.punctuation)\n",
    "\n",
    "df['punctuation_count'] = df['text'].apply(punctuation_count)\n",
    "\n",
    "# Upper case count\n",
    "def upper_case_count(text):\n",
    "    return sum(1 for char in text if char.isupper())\n",
    "\n",
    "df['upper_case_count'] = df['text'].apply(upper_case_count)\n",
    "\n",
    "def title_word_count(text):\n",
    "    return sum(1 for word in text.split() if word.istitle())\n",
    "\n",
    "df['title_word_count'] = df['text'].apply(title_word_count)\n",
    "\n",
    "# parts of speech\n",
    "def parts_of_speech(text):\n",
    "    pos_tags = pos_tag(word_tokenize(text))\n",
    "    \n",
    "    noun_count = sum(1 for tag in pos_tags if tag[1] in ['NN', 'NNS', 'NNP', 'NNPS'])\n",
    "    adv_count = sum(1 for tag in pos_tags if tag[1] in ['RB', 'RBR', 'RBS'])\n",
    "    verb_count = sum(1 for tag in pos_tags if tag[1] in ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ'])\n",
    "    adj_count = sum(1 for tag in pos_tags if tag[1] in ['JJ', 'JJR', 'JJS'])\n",
    "    pro_count = sum(1 for tag in pos_tags if tag[1] in ['PRP', 'PRP$', 'WP', 'WP$'])\n",
    "    return pd.Series([noun_count, adv_count, verb_count, adj_count, pro_count], index=['noun_count','adv_count','verb_count','adj_count','pro_count'])\n",
    "\n",
    "df[['noun_count','adv_count','verb_count','adj_count','pro_count']] = df['cleaned_text'].apply(lambda x: parts_of_speech(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Topic Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [text.split() for text in df['cleaned_text']]\n",
    "\n",
    "dictionary = corpora.Dictionary(corpus)\n",
    "\n",
    "corpus_bow = [dictionary.doc2bow(text) for text in corpus]\n",
    "\n",
    "# Training\n",
    "num_topics = 20\n",
    "lda_model = LdaModel(corpus_bow, num_topics=num_topics, id2word=dictionary, passes=15)\n",
    "\n",
    "topic_distribution = lda_model.get_document_topics(corpus_bow)\n",
    "\n",
    "for topic in range(num_topics):\n",
    "    df[f'topic_{topic + 1}_score'] = [next((t[1] for t in topic_dist if t[0] == topic), 0) for topic_dist in topic_distribution]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>char_count</th>\n",
       "      <th>word_count</th>\n",
       "      <th>word_density</th>\n",
       "      <th>punctuation_count</th>\n",
       "      <th>upper_case_count</th>\n",
       "      <th>title_word_count</th>\n",
       "      <th>noun_count</th>\n",
       "      <th>adv_count</th>\n",
       "      <th>...</th>\n",
       "      <th>topic_11_score</th>\n",
       "      <th>topic_12_score</th>\n",
       "      <th>topic_13_score</th>\n",
       "      <th>topic_14_score</th>\n",
       "      <th>topic_15_score</th>\n",
       "      <th>topic_16_score</th>\n",
       "      <th>topic_17_score</th>\n",
       "      <th>topic_18_score</th>\n",
       "      <th>topic_19_score</th>\n",
       "      <th>topic_20_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [text, cleaned_text, char_count, word_count, word_density, punctuation_count, upper_case_count, title_word_count, noun_count, adv_count, verb_count, adj_count, pro_count, topic_1_score, topic_2_score, topic_3_score, topic_4_score, topic_5_score, topic_6_score, topic_7_score, topic_8_score, topic_9_score, topic_10_score, topic_11_score, topic_12_score, topic_13_score, topic_14_score, topic_15_score, topic_16_score, topic_17_score, topic_18_score, topic_19_score, topic_20_score]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 33 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Readability Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['flesch_kincaid_score'] = df['cleaned_text'].apply(lambda x: textstat.flesch_kincaid_grade(x))\n",
    "\n",
    "df['flesch_score'] = df['cleaned_text'].apply(lambda x: textstat.flesch_reading_ease(x))\n",
    "\n",
    "df['gunning_fog_score'] = df['cleaned_text'].apply(lambda x: textstat.gunning_fog(x))\n",
    "\n",
    "df['coleman_liau_score'] = df['cleaned_text'].apply(lambda x: textstat.coleman_liau_index(x))\n",
    "\n",
    "df['dale_chall_score'] = df['cleaned_text'].apply(lambda x: textstat.dale_chall_readability_score(x))\n",
    "\n",
    "df['ari_score'] = df['cleaned_text'].apply(lambda x: textstat.automated_readability_index(x))\n",
    "\n",
    "df['linsear_write_score'] = df['cleaned_text'].apply(lambda x: textstat.linsear_write_formula(x))\n",
    "\n",
    "df['spache_score'] = df['cleaned_text'].apply(lambda x: textstat.spache_readability(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>char_count</th>\n",
       "      <th>word_count</th>\n",
       "      <th>word_density</th>\n",
       "      <th>punctuation_count</th>\n",
       "      <th>upper_case_count</th>\n",
       "      <th>title_word_count</th>\n",
       "      <th>noun_count</th>\n",
       "      <th>adv_count</th>\n",
       "      <th>...</th>\n",
       "      <th>topic_19_score</th>\n",
       "      <th>topic_20_score</th>\n",
       "      <th>flesch_kincaid_score</th>\n",
       "      <th>flesch_score</th>\n",
       "      <th>gunning_fog_score</th>\n",
       "      <th>coleman_liau_score</th>\n",
       "      <th>dale_chall_score</th>\n",
       "      <th>ari_score</th>\n",
       "      <th>linsear_write_score</th>\n",
       "      <th>spache_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [text, cleaned_text, char_count, word_count, word_density, punctuation_count, upper_case_count, title_word_count, noun_count, adv_count, verb_count, adj_count, pro_count, topic_1_score, topic_2_score, topic_3_score, topic_4_score, topic_5_score, topic_6_score, topic_7_score, topic_8_score, topic_9_score, topic_10_score, topic_11_score, topic_12_score, topic_13_score, topic_14_score, topic_15_score, topic_16_score, topic_17_score, topic_18_score, topic_19_score, topic_20_score, flesch_kincaid_score, flesch_score, gunning_fog_score, coleman_liau_score, dale_chall_score, ari_score, linsear_write_score, spache_score]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 41 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Named Entity Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ner_count(text):\n",
    "    words = word_tokenize(text)\n",
    "    pos_tags = pos_tag(words)\n",
    "    ner_tags = ne_chunk(pos_tags)\n",
    "    ner_count = sum(1 for chunk in ner_tags if hasattr(chunk, 'label'))\n",
    "    return ner_count\n",
    "\n",
    "df['ner_count'] = df['text'].apply(ner_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text error length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "tool = LanguageTool('en-US')\n",
    "\n",
    "def error_length(text):\n",
    "    matches = tool.check(text)\n",
    "    return len(matches)\n",
    "\n",
    "df['error_length'] = df['text'].apply(error_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>char_count</th>\n",
       "      <th>word_count</th>\n",
       "      <th>word_density</th>\n",
       "      <th>punctuation_count</th>\n",
       "      <th>upper_case_count</th>\n",
       "      <th>title_word_count</th>\n",
       "      <th>noun_count</th>\n",
       "      <th>...</th>\n",
       "      <th>flesch_kincaid_score</th>\n",
       "      <th>flesch_score</th>\n",
       "      <th>gunning_fog_score</th>\n",
       "      <th>coleman_liau_score</th>\n",
       "      <th>dale_chall_score</th>\n",
       "      <th>ari_score</th>\n",
       "      <th>linsear_write_score</th>\n",
       "      <th>spache_score</th>\n",
       "      <th>ner_count</th>\n",
       "      <th>error_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Face on Mars is nothing but a natural occu...</td>\n",
       "      <td>0</td>\n",
       "      <td>the face on mars is nothing but a natural occu...</td>\n",
       "      <td>1303</td>\n",
       "      <td>241</td>\n",
       "      <td>0.184958</td>\n",
       "      <td>64</td>\n",
       "      <td>42</td>\n",
       "      <td>31</td>\n",
       "      <td>65</td>\n",
       "      <td>...</td>\n",
       "      <td>94.9</td>\n",
       "      <td>-156.23</td>\n",
       "      <td>99.22</td>\n",
       "      <td>9.78</td>\n",
       "      <td>19.46</td>\n",
       "      <td>119.8</td>\n",
       "      <td>63.0</td>\n",
       "      <td>36.28</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Students have a higher chance of catching a vi...</td>\n",
       "      <td>0</td>\n",
       "      <td>students have a higher chance of catching a vi...</td>\n",
       "      <td>3456</td>\n",
       "      <td>605</td>\n",
       "      <td>0.175058</td>\n",
       "      <td>62</td>\n",
       "      <td>40</td>\n",
       "      <td>39</td>\n",
       "      <td>149</td>\n",
       "      <td>...</td>\n",
       "      <td>236.9</td>\n",
       "      <td>-525.69</td>\n",
       "      <td>243.26</td>\n",
       "      <td>11.52</td>\n",
       "      <td>35.60</td>\n",
       "      <td>303.3</td>\n",
       "      <td>58.0</td>\n",
       "      <td>86.91</td>\n",
       "      <td>4</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Driverless cars have good and bad things that ...</td>\n",
       "      <td>0</td>\n",
       "      <td>driverless cars have good and bad things that ...</td>\n",
       "      <td>1629</td>\n",
       "      <td>305</td>\n",
       "      <td>0.187231</td>\n",
       "      <td>26</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>55</td>\n",
       "      <td>...</td>\n",
       "      <td>118.7</td>\n",
       "      <td>-212.73</td>\n",
       "      <td>123.44</td>\n",
       "      <td>9.37</td>\n",
       "      <td>20.78</td>\n",
       "      <td>151.5</td>\n",
       "      <td>53.0</td>\n",
       "      <td>44.69</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Some people might think that traveling in a gr...</td>\n",
       "      <td>1</td>\n",
       "      <td>some people might think that traveling in a gr...</td>\n",
       "      <td>328</td>\n",
       "      <td>67</td>\n",
       "      <td>0.204268</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>24.7</td>\n",
       "      <td>37.31</td>\n",
       "      <td>28.59</td>\n",
       "      <td>6.58</td>\n",
       "      <td>8.61</td>\n",
       "      <td>30.5</td>\n",
       "      <td>38.5</td>\n",
       "      <td>10.93</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>How many of us students want to be forced to d...</td>\n",
       "      <td>0</td>\n",
       "      <td>how many of us students want to be forced to d...</td>\n",
       "      <td>3431</td>\n",
       "      <td>612</td>\n",
       "      <td>0.178374</td>\n",
       "      <td>62</td>\n",
       "      <td>27</td>\n",
       "      <td>25</td>\n",
       "      <td>133</td>\n",
       "      <td>...</td>\n",
       "      <td>240.8</td>\n",
       "      <td>-541.25</td>\n",
       "      <td>247.35</td>\n",
       "      <td>10.94</td>\n",
       "      <td>36.26</td>\n",
       "      <td>306.3</td>\n",
       "      <td>64.0</td>\n",
       "      <td>88.21</td>\n",
       "      <td>4</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86581</th>\n",
       "      <td>Dear ( Senator of Florida ) , In my opinion I ...</td>\n",
       "      <td>0</td>\n",
       "      <td>dear senator of florida in my opinion i believ...</td>\n",
       "      <td>2550</td>\n",
       "      <td>477</td>\n",
       "      <td>0.187059</td>\n",
       "      <td>31</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>185.8</td>\n",
       "      <td>-387.31</td>\n",
       "      <td>192.56</td>\n",
       "      <td>9.43</td>\n",
       "      <td>29.58</td>\n",
       "      <td>237.6</td>\n",
       "      <td>57.0</td>\n",
       "      <td>69.07</td>\n",
       "      <td>11</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86582</th>\n",
       "      <td>Dear Principal : I think we should have cell p...</td>\n",
       "      <td>0</td>\n",
       "      <td>dear principal i think we should have cell pho...</td>\n",
       "      <td>1718</td>\n",
       "      <td>334</td>\n",
       "      <td>0.194412</td>\n",
       "      <td>35</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>77</td>\n",
       "      <td>...</td>\n",
       "      <td>128.8</td>\n",
       "      <td>-233.70</td>\n",
       "      <td>135.04</td>\n",
       "      <td>8.27</td>\n",
       "      <td>22.00</td>\n",
       "      <td>165.1</td>\n",
       "      <td>59.0</td>\n",
       "      <td>48.50</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86583</th>\n",
       "      <td>Dear Teacher_NAME I think that if you try to s...</td>\n",
       "      <td>0</td>\n",
       "      <td>dear teachername i think that if you try to st...</td>\n",
       "      <td>1516</td>\n",
       "      <td>267</td>\n",
       "      <td>0.176121</td>\n",
       "      <td>27</td>\n",
       "      <td>36</td>\n",
       "      <td>20</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>103.9</td>\n",
       "      <td>-174.16</td>\n",
       "      <td>108.45</td>\n",
       "      <td>11.34</td>\n",
       "      <td>18.89</td>\n",
       "      <td>134.1</td>\n",
       "      <td>51.0</td>\n",
       "      <td>39.26</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86584</th>\n",
       "      <td>Venus is sometimes called the `` meaning Star ...</td>\n",
       "      <td>0</td>\n",
       "      <td>venus is sometimes called the meaning star its...</td>\n",
       "      <td>878</td>\n",
       "      <td>166</td>\n",
       "      <td>0.189066</td>\n",
       "      <td>21</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>41</td>\n",
       "      <td>...</td>\n",
       "      <td>64.5</td>\n",
       "      <td>-71.64</td>\n",
       "      <td>68.33</td>\n",
       "      <td>8.84</td>\n",
       "      <td>15.39</td>\n",
       "      <td>81.8</td>\n",
       "      <td>56.0</td>\n",
       "      <td>25.75</td>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86585</th>\n",
       "      <td>The Seagoing Cowboy Bros Do you like going to ...</td>\n",
       "      <td>0</td>\n",
       "      <td>the seagoing cowboy bros do you like going to ...</td>\n",
       "      <td>1987</td>\n",
       "      <td>362</td>\n",
       "      <td>0.182184</td>\n",
       "      <td>64</td>\n",
       "      <td>77</td>\n",
       "      <td>65</td>\n",
       "      <td>100</td>\n",
       "      <td>...</td>\n",
       "      <td>142.1</td>\n",
       "      <td>-279.04</td>\n",
       "      <td>146.57</td>\n",
       "      <td>10.24</td>\n",
       "      <td>24.51</td>\n",
       "      <td>180.7</td>\n",
       "      <td>53.0</td>\n",
       "      <td>52.83</td>\n",
       "      <td>21</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>86586 rows × 44 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label  \\\n",
       "0      The Face on Mars is nothing but a natural occu...      0   \n",
       "1      Students have a higher chance of catching a vi...      0   \n",
       "2      Driverless cars have good and bad things that ...      0   \n",
       "3      Some people might think that traveling in a gr...      1   \n",
       "4      How many of us students want to be forced to d...      0   \n",
       "...                                                  ...    ...   \n",
       "86581  Dear ( Senator of Florida ) , In my opinion I ...      0   \n",
       "86582  Dear Principal : I think we should have cell p...      0   \n",
       "86583  Dear Teacher_NAME I think that if you try to s...      0   \n",
       "86584  Venus is sometimes called the `` meaning Star ...      0   \n",
       "86585  The Seagoing Cowboy Bros Do you like going to ...      0   \n",
       "\n",
       "                                            cleaned_text  char_count  \\\n",
       "0      the face on mars is nothing but a natural occu...        1303   \n",
       "1      students have a higher chance of catching a vi...        3456   \n",
       "2      driverless cars have good and bad things that ...        1629   \n",
       "3      some people might think that traveling in a gr...         328   \n",
       "4      how many of us students want to be forced to d...        3431   \n",
       "...                                                  ...         ...   \n",
       "86581  dear senator of florida in my opinion i believ...        2550   \n",
       "86582  dear principal i think we should have cell pho...        1718   \n",
       "86583  dear teachername i think that if you try to st...        1516   \n",
       "86584  venus is sometimes called the meaning star its...         878   \n",
       "86585  the seagoing cowboy bros do you like going to ...        1987   \n",
       "\n",
       "       word_count  word_density  punctuation_count  upper_case_count  \\\n",
       "0             241      0.184958                 64                42   \n",
       "1             605      0.175058                 62                40   \n",
       "2             305      0.187231                 26                22   \n",
       "3              67      0.204268                  6                 5   \n",
       "4             612      0.178374                 62                27   \n",
       "...           ...           ...                ...               ...   \n",
       "86581         477      0.187059                 31                30   \n",
       "86582         334      0.194412                 35                17   \n",
       "86583         267      0.176121                 27                36   \n",
       "86584         166      0.189066                 21                17   \n",
       "86585         362      0.182184                 64                77   \n",
       "\n",
       "       title_word_count  noun_count  ...  flesch_kincaid_score  flesch_score  \\\n",
       "0                    31          65  ...                  94.9       -156.23   \n",
       "1                    39         149  ...                 236.9       -525.69   \n",
       "2                    22          55  ...                 118.7       -212.73   \n",
       "3                     5          11  ...                  24.7         37.31   \n",
       "4                    25         133  ...                 240.8       -541.25   \n",
       "...                 ...         ...  ...                   ...           ...   \n",
       "86581                30          94  ...                 185.8       -387.31   \n",
       "86582                15          77  ...                 128.8       -233.70   \n",
       "86583                20          60  ...                 103.9       -174.16   \n",
       "86584                17          41  ...                  64.5        -71.64   \n",
       "86585                65         100  ...                 142.1       -279.04   \n",
       "\n",
       "       gunning_fog_score  coleman_liau_score  dale_chall_score  ari_score  \\\n",
       "0                  99.22                9.78             19.46      119.8   \n",
       "1                 243.26               11.52             35.60      303.3   \n",
       "2                 123.44                9.37             20.78      151.5   \n",
       "3                  28.59                6.58              8.61       30.5   \n",
       "4                 247.35               10.94             36.26      306.3   \n",
       "...                  ...                 ...               ...        ...   \n",
       "86581             192.56                9.43             29.58      237.6   \n",
       "86582             135.04                8.27             22.00      165.1   \n",
       "86583             108.45               11.34             18.89      134.1   \n",
       "86584              68.33                8.84             15.39       81.8   \n",
       "86585             146.57               10.24             24.51      180.7   \n",
       "\n",
       "       linsear_write_score  spache_score  ner_count  error_length  \n",
       "0                     63.0         36.28         12            31  \n",
       "1                     58.0         86.91          4            60  \n",
       "2                     53.0         44.69          5            21  \n",
       "3                     38.5         10.93          0             6  \n",
       "4                     64.0         88.21          4            60  \n",
       "...                    ...           ...        ...           ...  \n",
       "86581                 57.0         69.07         11            34  \n",
       "86582                 59.0         48.50          1            34  \n",
       "86583                 51.0         39.26          1            27  \n",
       "86584                 56.0         25.75          8            17  \n",
       "86585                 53.0         52.83         21            48  \n",
       "\n",
       "[86586 rows x 44 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vect = df['cleaned_text'].tolist()\n",
    "\n",
    "count_vectorizer = CountVectorizer(max_features=5000)\n",
    "\n",
    "count_matrix = count_vectorizer.fit_transform(count_vect)\n",
    "\n",
    "count_df = pd.DataFrame(count_matrix.toarray(), columns=count_vectorizer.get_feature_names_out())\n",
    "\n",
    "df = pd.concat([df, count_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joblib.dump(count_vectorizer, 'tools/count_vectorizer_50k.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bigram Vectorizer\n",
    "\n",
    "bigram_vect = df['cleaned_text'].tolist()\n",
    "\n",
    "bigram_vectorizer = TfidfVectorizer(ngram_range=(2, 2), max_features=5000)\n",
    "\n",
    "bigram_matrix = bigram_vectorizer.fit_transform(bigram_vect)\n",
    "\n",
    "bigram_df = pd.DataFrame(bigram_matrix.toarray(), columns=bigram_vectorizer.get_feature_names_out())\n",
    "\n",
    "df = pd.concat([df, bigram_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joblib.dump(bigram_vectorizer, 'tools/bigram_vectorizer_50k.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trigram Vectorizer\n",
    "\n",
    "trigram_vect = df['cleaned_text'].tolist()\n",
    "\n",
    "trigram_vectorizer = TfidfVectorizer(ngram_range=(3, 3), max_features=5000)\n",
    "\n",
    "trigram_matrix = trigram_vectorizer.fit_transform(trigram_vect)\n",
    "\n",
    "trigram_df = pd.DataFrame(trigram_matrix.toarray(), columns=trigram_vectorizer.get_feature_names_out())\n",
    "\n",
    "df = pd.concat([df, trigram_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joblib.dump(trigram_vectorizer, 'tools/trigram_vectorizer_50k.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bi-Trigram Vectorizer\n",
    "\n",
    "bitri_vect = df['cleaned_text'].str.strip().tolist()\n",
    "\n",
    "bitri_vectorizer = TfidfVectorizer(analyzer='char', ngram_range=(2, 3), max_features=5000)\n",
    "\n",
    "bichar_matrix = bitri_vectorizer.fit_transform(bitri_vect)\n",
    "\n",
    "bichar_df = pd.DataFrame(bichar_matrix.toarray(), columns=bitri_vectorizer.get_feature_names_out())\n",
    "\n",
    "bichar_df.columns = bichar_df.columns.str.strip()\n",
    "\n",
    "bichar_df = bichar_df.loc[:, ~bichar_df.columns.duplicated()]\n",
    "\n",
    "df = pd.concat([df, bichar_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joblib.dump(bitri_vectorizer, 'tools/bitri_vectorizer_v2.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lexical Diversity\n",
    "\n",
    "ttr_list = [len(set(word_tokenize(text.lower()))) / len(word_tokenize(text.lower())) for text in df['cleaned_text']]\n",
    "\n",
    "ttr_df = pd.DataFrame({'lexical_diversity': ttr_list})\n",
    "\n",
    "df['lexical_diversity'] = ttr_df['lexical_diversity']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('DATA_SET/final_test_v2.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}