vectorization/Vectorization_Example_Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from vectorization_nlp import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read Data and eliminate null values\n",
    "df = pd.read_excel(\"Data.xlsx\")\n",
    "df.dropna(inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color='red'>Note: Press <Shift+Tab> to access the docstring of <B><I>various functions</B></I>, which contains various default parameters for processing</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: count_vectorizer (Params - (Strings_to_be_processed, Dictionary of function related parameters)\n",
    "# Function: custom_tokenizer is inbuilt (Where sentences are seperated by ' . ')\n",
    "# Returns object of CountVectorizer and Tranformed sentences (Scikit-learn)\n",
    "\n",
    "cv_obj, cv_sent = count_vectorizer(df['text'].tolist(), {'tokenizer': custom_tokenizer})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample test doc\n",
    "doc = [\"this is to test the vectorization technique and some random words here and there\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: tfidf_vectorizer (Params - (Strings_to_be_processed, Dictionary of function related parameters)\n",
    "# Function: custom_tokenizer is inbuilt (Where sentences are seperated by ' . ')\n",
    "# Returns object of tfidfVectorizer and Tranformed sentences (Scikit-learn)\n",
    "\n",
    "tf_obj, tf_sent = tfidf_vectorizer(df['text'].tolist(), {'tokenizer': custom_tokenizer})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: top_words_tfidf (Params - (TFIDF_Object, List_of_strings, number_of_top_words)\n",
    "# Prints the Top Words (Highest IDF) for a given document according to the object passed\n",
    "# Returns a dictionary with keys as words, and values as IDF value\n",
    "\n",
    "top_words_tfidf(tf_obj, doc, 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample test doc\n",
    "doc = [\"this is to test the vetorization\".split(), \"technique and some random words here and there\".split()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Train Model*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: train_word2vec (Params - (list_of_list (words))\n",
    "# Ex - [['here', 'there'], ['clear', 'tear']]\n",
    "# Returns Gensim Word2Vec Model\n",
    "model = train_word2vec(doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*(or) Load pretrained model*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: load_word2vec (Params - (path_of_file)\n",
    "# Returns Gensim Word2Vec Model\n",
    "w2v_model = load_word2vec(\"GoogleNews-vectors-negative300.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the most_similar and similarity score for a given word or pair\n",
    "print(\"Most Similar:\", get_most_similar(w2v_model, 'is'))\n",
    "print(\"Similarity Score:\", get_similarity_score(w2v_model, 'the', 'was'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### GloVe "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Load pretrained model*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function: load_glove (Params - (path_of_file)\n",
    "# Returns Gensim Word2Vec Model (Converts GloVe file to word2vec)\n",
    "glove_model = load_glove('glove.6B/glove.6B.100d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the most_similar and similarity score for a given word or pair\n",
    "print(\"Most Similar:\", get_most_similar(glove_model, 'is'))\n",
    "print(\"Similarity Score:\", get_similarity_score(glove_model, 'the', 'was'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Get average of Word2Vec or GloVe in each sentence, and represent it in one vector*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_wise_vectors = get_sentence_wise_vector(glove_model, doc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}