Results_randReviews_10topics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### READ RESULTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "### WE SHOULD HAVE 3 DIFF SAMPLES\n",
    "path_data_file = \"../DataProject/reviews_clean.p\"\n",
    "\n",
    "name_model_pickle_lda = \"u_lda10k_lda.p\"\n",
    "name_model_pickle_slda = \"r_slda10k_slda.p\"\n",
    "name_model_pickle_dmr = \"r_dmr10k_dmr.p\"\n",
    "\n",
    "num_words_to_compare = 10\n",
    "\n",
    "import pandas as pd\n",
    "import lda\n",
    "\n",
    "## You should have these scripts\n",
    "import vocabulary as v\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "data = pd.read_pickle(path_data_file)\n",
    "\n",
    "result_lda = pd.read_pickle(name_model_pickle_lda)\n",
    "result_slda = pd.read_pickle(name_model_pickle_lda)\n",
    "result_dmr = pd.read_pickle(name_model_pickle_lda)\n",
    "\n",
    "voca = v.Vocabulary()\n",
    "docs = voca.read_corpus( data['text'] )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA OUTPUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- LDA OUTPUT\n",
      "Alpha 0.1\n",
      "Beta 0.01\n",
      "Docs with words 7993\n",
      "10 topics per doc 7993\n",
      "34174 word-assignment per topic 10\n",
      "Voc size 37929\n"
     ]
    }
   ],
   "source": [
    "print \"--- LDA OUTPUT\"\n",
    "\n",
    "print \"Alpha\", result_lda.alpha\n",
    "print \"Beta\", result_lda.beta\n",
    "print \"Docs with words\", len(result_lda.docs)\n",
    "print len(result_lda.topicdist()[0]), \"topics per doc\", len(result_lda.topicdist())\n",
    "print len(result_lda.worddist()[0]), \"word-assignment per topic\", len(result_lda.worddist())\n",
    "print \"Voc size\", voca.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Example of document training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- S T A R T   E X A M P L E\n",
      "Text\n",
      "[u'3', u'5', u'stars', u'nice', u'bar', u'flew', u'met', u'friends', u'night', u'since', u'getting', u'married', u'next', u'day', u'probably', u'20', u'us', u'dressed', u'looking', u'like', u'got', u'plane', u'cool', u'setup', u'water', u'features', u'place', u'fiber', u'optics', u'service', u'great', u'glasses', u'wine', u'know', u'beer', u'guy', u'relaxed', u'hung', u'midnight', u'service', u'kept', u'us', u'admit', u'cool', u'setup', u'one', u'thing', u'since', u'pre', u'wedding', u'sort', u'party', u'idea', u'prices', u'like', u'fun', u'laid', u'back', u'time', u'friends', u'good', u'place'] \n",
      "\n",
      "31 ' Words\n",
      "[1695, 229, 11329, 11813, 2437, 127, 107, 2673, 2384, 99, 2065, 54, 248, 119, 230, 1177, 390, 665, 1164, 194, 823, 34, 1251, 119, 479, 99, 21, 40, 386, 3305, 12202] \n",
      "\n",
      "10 ' Topic probabilities\n",
      "[ 0.003125  0.003125  0.503125  0.003125  0.003125  0.065625  0.065625\n",
      "  0.253125  0.096875  0.003125] \n",
      "\n",
      "Topic-assignment sorted:\n",
      "\n",
      "[(0.503125, 2.0) (0.25312500000000004, 7.0) (0.09687500000000004, 8.0)\n",
      " (0.06562500000000004, 6.0) (0.06562499999999999, 5.0)\n",
      " (0.0031250000000000444, 3.0) (0.0031250000000000444, 1.0)\n",
      " (0.0031250000000000444, 0.0) (0.0031250000000000028, 4.0)\n",
      " (0.003124999999999989, 9.0)]\n",
      "---- E N D   E X A M P L E\n"
     ]
    }
   ],
   "source": [
    "\n",
    "example = np.random.randint(0,len(result_lda.docs))\n",
    "\n",
    "print \"--- S T A R T   E X A M P L E\"\n",
    "print \"Text\\n\",data.text[example], \"\\n\"\n",
    "print len(result_lda.docs[example]), \"' Words\\n\", result_lda.docs[example], \"\\n\"\n",
    "print len(result_lda.topicdist()[example]), \"' Topic probabilities\\n\", result_lda.topicdist()[example], \"\\n\"\n",
    "\n",
    "print \"Topic-assignment sorted:\\n\"\n",
    "dtype_list = [('weight', float), ('topic', float)] \n",
    "topic_assign = np.array( zip(result_lda.topicdist()[example],range(0,len(result_lda.topicdist()[example]))), \n",
    "                        dtype=dtype_list)\n",
    "topic_assign = np.sort(topic_assign,order='weight')[::-1]\n",
    "print topic_assign\n",
    "print \"---- E N D   E X A M P L E\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def compute_distances(dataset, distance_measures, n, column = 'index', name=0 ):\n",
    "    '''\n",
    "    dataset -- weights of words (same length of columns in each cluster)\n",
    "    \n",
    "    name -- name of cluster (number assignment)\n",
    "    \n",
    "    distance_measures -- list of [ ‘braycurtis’, ‘canberra’, ‘chebyshev’, ‘cityblock’, ‘correlation’, ‘cosine’, ‘dice’, \n",
    "    ‘euclidean’, ‘hamming’, ‘jaccard’, ‘kulsinski’, ‘mahalanobis’, ‘matching’, ‘minkowski’, ‘rogerstanimoto’, \n",
    "    ‘russellrao’, ‘seuclidean’, ‘sokalmichener’, ‘sokalsneath’, ‘sqeuclidean’, ‘yule’ ]\n",
    "    \n",
    "    n -- number of most similar that we want to get\n",
    "    \n",
    "    Example: compute_distance('id_example', ['euclidean'], 6)\n",
    "    \n",
    "    '''\n",
    "    distances = pd.DataFrame()\n",
    "    from scipy.spatial import distance\n",
    "    from scipy.spatial.distance import pdist, squareform\n",
    "    \n",
    "    ## Find the location (row) - topic - we are looking for\n",
    "    ## (include \"name in parameters if we want two particuar rows of a dataframe)\n",
    "    if column == 'index':\n",
    "        id_location = np.where(dataset.index == name)[0][0]\n",
    "    else:\n",
    "        id_location = np.where(dataset[column] == name)[0][0]\n",
    "\n",
    "    # Go through all distance measures we care about\n",
    "    print n, \"' Clusters that are closer to topic =\", name\n",
    "    print \"Format: (cluster number, similarity measure)\"\n",
    "    \n",
    "    for distance_measure in distance_measures:\n",
    "    \n",
    "        # Find all pairwise distances\n",
    "        current_distances = distance.squareform(distance.pdist(dataset, distance_measure))\n",
    "        # Get the closest n elements for the whiskey we care about\n",
    "        most_similar = np.argsort(current_distances[:, id_location])[1:n+1]\n",
    "        # Append results (a new column to the dataframe with the name of the measure)\n",
    "        distances[distance_measure] = list(zip(data.index[most_similar], current_distances[most_similar, id_location]))\n",
    "        \n",
    "    return distances\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Similarity and spearman different words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "topwords = result_lda.word_dist_with_voca(voca, topk=num_words_to_compare)\n",
    "\n",
    "#for i in topwords: print i, \"TOP WORDS\", topwords[i],\"\\n\"\n",
    "        \n",
    "columns_names = ['W_lda_'+str(x) for x in range(num_words_to_compare)]\n",
    "columns_names.insert(0,'Cluster_lda')\n",
    "\n",
    "weights_topwords = pd.DataFrame(columns=columns_names).T\n",
    "weight_words_topwords = pd.DataFrame(columns=columns_names).T\n",
    "dataframe_topwords = pd.DataFrame(columns=columns_names).T\n",
    "\n",
    "import operator\n",
    "\n",
    "for i in range(result_lda.K):\n",
    "    weight_list_i = list(np.sort(topwords[i].values())[::-1])\n",
    "    weight_list_i.insert(0,i)\n",
    "    weights_topwords[i] = weight_list_i\n",
    "    word_list_i = list(np.sort(topwords[i].keys()))\n",
    "    word_list_i.insert(0,i)\n",
    "    dataframe_topwords[i] = word_list_i\n",
    "    word_weights_i = list([ x[0] for x in sorted(topwords[i].items(), key=operator.itemgetter(1),reverse=1) ])\n",
    "    word_weights_i.insert(0,i)\n",
    "    weight_words_topwords[i] = word_weights_i\n",
    "    \n",
    "weights_topwords = weights_topwords.T\n",
    "dataframe_topwords = dataframe_topwords.T\n",
    "weight_words_topwords = weight_words_topwords.T\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 ' Clusters that are closer to topic = 1\n",
      "Format: (cluster number, similarity measure)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>euclidean</th>\n",
       "      <th>cosine</th>\n",
       "      <th>correlation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(0, 1.00004737315)</td>\n",
       "      <td>(2, 0.0001512739302)</td>\n",
       "      <td>(2, 9.75651190301e-05)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(2, 1.00015544675)</td>\n",
       "      <td>(4, 0.000232582333328)</td>\n",
       "      <td>(4, 0.000104704497876)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(3, 2.00002143401)</td>\n",
       "      <td>(3, 0.000353984133143)</td>\n",
       "      <td>(3, 0.000119601619214)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(4, 3.00015258257)</td>\n",
       "      <td>(9, 0.000478700197643)</td>\n",
       "      <td>(9, 0.000145457158564)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(5, 4.00004882693)</td>\n",
       "      <td>(8, 0.00053398772975)</td>\n",
       "      <td>(8, 0.00016099952919)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            euclidean                  cosine             correlation\n",
       "0  (0, 1.00004737315)    (2, 0.0001512739302)  (2, 9.75651190301e-05)\n",
       "1  (2, 1.00015544675)  (4, 0.000232582333328)  (4, 0.000104704497876)\n",
       "2  (3, 2.00002143401)  (3, 0.000353984133143)  (3, 0.000119601619214)\n",
       "3  (4, 3.00015258257)  (9, 0.000478700197643)  (9, 0.000145457158564)\n",
       "4  (5, 4.00004882693)   (8, 0.00053398772975)   (8, 0.00016099952919)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#### This dataset CALLED \"WEIGHT_TOPWORDS\" needs to have weights of words on diff clusters \n",
    "#### We want to compare each of this clusters not with themselves (LDA) BUT with the others\n",
    "#### EXAMPLE: \"WEIGHTS OF LDA CLUSTER-1\"   AGAINST   \"ALL CLUSTERS  -1 TO 10- OF SLDA/DMR WEIGHTS\"\n",
    "#### IN ORDER TO FIGURE OUT WHO IS THE MOST SIMILAR IN ANOTHER OUTPUT !!!!\n",
    "\n",
    "compute_distances(weights_topwords, ['euclidean','cosine','correlation'], 5, column='Cluster_lda', name=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.018050</td>\n",
       "      <td>0.013020</td>\n",
       "      <td>0.010459</td>\n",
       "      <td>0.010278</td>\n",
       "      <td>0.009769</td>\n",
       "      <td>0.007500</td>\n",
       "      <td>0.007318</td>\n",
       "      <td>0.007173</td>\n",
       "      <td>0.007082</td>\n",
       "      <td>0.006465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.027067</td>\n",
       "      <td>0.009914</td>\n",
       "      <td>0.009862</td>\n",
       "      <td>0.009416</td>\n",
       "      <td>0.008288</td>\n",
       "      <td>0.008105</td>\n",
       "      <td>0.007501</td>\n",
       "      <td>0.007265</td>\n",
       "      <td>0.006977</td>\n",
       "      <td>0.006741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.024266</td>\n",
       "      <td>0.024034</td>\n",
       "      <td>0.019381</td>\n",
       "      <td>0.011874</td>\n",
       "      <td>0.010038</td>\n",
       "      <td>0.009539</td>\n",
       "      <td>0.008077</td>\n",
       "      <td>0.007809</td>\n",
       "      <td>0.007738</td>\n",
       "      <td>0.007595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.018924</td>\n",
       "      <td>0.013867</td>\n",
       "      <td>0.010043</td>\n",
       "      <td>0.008557</td>\n",
       "      <td>0.007670</td>\n",
       "      <td>0.007383</td>\n",
       "      <td>0.006676</td>\n",
       "      <td>0.006568</td>\n",
       "      <td>0.006400</td>\n",
       "      <td>0.005945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.035467</td>\n",
       "      <td>0.023563</td>\n",
       "      <td>0.023316</td>\n",
       "      <td>0.019622</td>\n",
       "      <td>0.018062</td>\n",
       "      <td>0.017734</td>\n",
       "      <td>0.016420</td>\n",
       "      <td>0.013629</td>\n",
       "      <td>0.012972</td>\n",
       "      <td>0.012151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5.0</td>\n",
       "      <td>0.008704</td>\n",
       "      <td>0.007675</td>\n",
       "      <td>0.007174</td>\n",
       "      <td>0.006805</td>\n",
       "      <td>0.005935</td>\n",
       "      <td>0.005328</td>\n",
       "      <td>0.005328</td>\n",
       "      <td>0.005012</td>\n",
       "      <td>0.004484</td>\n",
       "      <td>0.004484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.011822</td>\n",
       "      <td>0.010010</td>\n",
       "      <td>0.009786</td>\n",
       "      <td>0.007825</td>\n",
       "      <td>0.007769</td>\n",
       "      <td>0.007284</td>\n",
       "      <td>0.007265</td>\n",
       "      <td>0.007209</td>\n",
       "      <td>0.006107</td>\n",
       "      <td>0.006051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0.010069</td>\n",
       "      <td>0.009685</td>\n",
       "      <td>0.009398</td>\n",
       "      <td>0.009075</td>\n",
       "      <td>0.007802</td>\n",
       "      <td>0.007497</td>\n",
       "      <td>0.006896</td>\n",
       "      <td>0.006878</td>\n",
       "      <td>0.006347</td>\n",
       "      <td>0.006303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8.0</td>\n",
       "      <td>0.018850</td>\n",
       "      <td>0.013057</td>\n",
       "      <td>0.012547</td>\n",
       "      <td>0.011196</td>\n",
       "      <td>0.009245</td>\n",
       "      <td>0.007954</td>\n",
       "      <td>0.007744</td>\n",
       "      <td>0.007384</td>\n",
       "      <td>0.006874</td>\n",
       "      <td>0.006514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0.032193</td>\n",
       "      <td>0.025738</td>\n",
       "      <td>0.018239</td>\n",
       "      <td>0.016525</td>\n",
       "      <td>0.012240</td>\n",
       "      <td>0.011892</td>\n",
       "      <td>0.010660</td>\n",
       "      <td>0.009990</td>\n",
       "      <td>0.009937</td>\n",
       "      <td>0.009910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cluster_lda   W_lda_0   W_lda_1   W_lda_2   W_lda_3   W_lda_4   W_lda_5  \\\n",
       "0          0.0  0.018050  0.013020  0.010459  0.010278  0.009769  0.007500   \n",
       "1          1.0  0.027067  0.009914  0.009862  0.009416  0.008288  0.008105   \n",
       "2          2.0  0.024266  0.024034  0.019381  0.011874  0.010038  0.009539   \n",
       "3          3.0  0.018924  0.013867  0.010043  0.008557  0.007670  0.007383   \n",
       "4          4.0  0.035467  0.023563  0.023316  0.019622  0.018062  0.017734   \n",
       "5          5.0  0.008704  0.007675  0.007174  0.006805  0.005935  0.005328   \n",
       "6          6.0  0.011822  0.010010  0.009786  0.007825  0.007769  0.007284   \n",
       "7          7.0  0.010069  0.009685  0.009398  0.009075  0.007802  0.007497   \n",
       "8          8.0  0.018850  0.013057  0.012547  0.011196  0.009245  0.007954   \n",
       "9          9.0  0.032193  0.025738  0.018239  0.016525  0.012240  0.011892   \n",
       "\n",
       "    W_lda_6   W_lda_7   W_lda_8   W_lda_9  \n",
       "0  0.007318  0.007173  0.007082  0.006465  \n",
       "1  0.007501  0.007265  0.006977  0.006741  \n",
       "2  0.008077  0.007809  0.007738  0.007595  \n",
       "3  0.006676  0.006568  0.006400  0.005945  \n",
       "4  0.016420  0.013629  0.012972  0.012151  \n",
       "5  0.005328  0.005012  0.004484  0.004484  \n",
       "6  0.007265  0.007209  0.006107  0.006051  \n",
       "7  0.006896  0.006878  0.006347  0.006303  \n",
       "8  0.007744  0.007384  0.006874  0.006514  \n",
       "9  0.010660  0.009990  0.009937  0.009910  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "weights_topwords \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>recommended</td>\n",
       "      <td>however</td>\n",
       "      <td>chair</td>\n",
       "      <td>setting</td>\n",
       "      <td>couches</td>\n",
       "      <td>months</td>\n",
       "      <td>men</td>\n",
       "      <td>fen</td>\n",
       "      <td>great</td>\n",
       "      <td>needed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>text</td>\n",
       "      <td>told</td>\n",
       "      <td>chair</td>\n",
       "      <td>stevenonmill</td>\n",
       "      <td>retro</td>\n",
       "      <td>great</td>\n",
       "      <td>recommended</td>\n",
       "      <td>def</td>\n",
       "      <td>finding</td>\n",
       "      <td>perfect</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>batter</td>\n",
       "      <td>recommended</td>\n",
       "      <td>couches</td>\n",
       "      <td>great</td>\n",
       "      <td>service</td>\n",
       "      <td>back</td>\n",
       "      <td>months</td>\n",
       "      <td>wait</td>\n",
       "      <td>ignored</td>\n",
       "      <td>needed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>batter</td>\n",
       "      <td>couches</td>\n",
       "      <td>brecksville</td>\n",
       "      <td>cut</td>\n",
       "      <td>chair</td>\n",
       "      <td>steamed</td>\n",
       "      <td>mercedes</td>\n",
       "      <td>sever</td>\n",
       "      <td>recommended</td>\n",
       "      <td>brews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>gab</td>\n",
       "      <td>shops</td>\n",
       "      <td>massieve</td>\n",
       "      <td>breakfast</td>\n",
       "      <td>realize</td>\n",
       "      <td>juicy</td>\n",
       "      <td>provided</td>\n",
       "      <td>humble</td>\n",
       "      <td>shawarma</td>\n",
       "      <td>dinning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>deserves</td>\n",
       "      <td>great</td>\n",
       "      <td>hair</td>\n",
       "      <td>imdb</td>\n",
       "      <td>estate</td>\n",
       "      <td>frazzled</td>\n",
       "      <td>second</td>\n",
       "      <td>def</td>\n",
       "      <td>dashed</td>\n",
       "      <td>wanted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>great</td>\n",
       "      <td>texting</td>\n",
       "      <td>time</td>\n",
       "      <td>frock</td>\n",
       "      <td>mom</td>\n",
       "      <td>recommend</td>\n",
       "      <td>service</td>\n",
       "      <td>would</td>\n",
       "      <td>back</td>\n",
       "      <td>months</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>def</td>\n",
       "      <td>would</td>\n",
       "      <td>months</td>\n",
       "      <td>time</td>\n",
       "      <td>specific</td>\n",
       "      <td>noticed</td>\n",
       "      <td>chair</td>\n",
       "      <td>selection</td>\n",
       "      <td>metal</td>\n",
       "      <td>even</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>witnessed</td>\n",
       "      <td>hey</td>\n",
       "      <td>chair</td>\n",
       "      <td>recommended</td>\n",
       "      <td>son</td>\n",
       "      <td>group</td>\n",
       "      <td>girl</td>\n",
       "      <td>sashimi</td>\n",
       "      <td>couches</td>\n",
       "      <td>wraps</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>somewhat</td>\n",
       "      <td>mouth</td>\n",
       "      <td>25</td>\n",
       "      <td>alfresco</td>\n",
       "      <td>boy</td>\n",
       "      <td>japanese</td>\n",
       "      <td>vowed</td>\n",
       "      <td>rabe</td>\n",
       "      <td>indicating</td>\n",
       "      <td>takes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cluster_lda      W_lda_0      W_lda_1      W_lda_2       W_lda_3   W_lda_4  \\\n",
       "0           0  recommended      however        chair       setting   couches   \n",
       "1           1         text         told        chair  stevenonmill     retro   \n",
       "2           2       batter  recommended      couches         great   service   \n",
       "3           3       batter      couches  brecksville           cut     chair   \n",
       "4           4          gab        shops     massieve     breakfast   realize   \n",
       "5           5     deserves        great         hair          imdb    estate   \n",
       "6           6        great      texting         time         frock       mom   \n",
       "7           7          def        would       months          time  specific   \n",
       "8           8    witnessed          hey        chair   recommended       son   \n",
       "9           9     somewhat        mouth           25      alfresco       boy   \n",
       "\n",
       "     W_lda_5      W_lda_6    W_lda_7      W_lda_8  W_lda_9  \n",
       "0     months          men        fen        great   needed  \n",
       "1      great  recommended        def      finding  perfect  \n",
       "2       back       months       wait      ignored   needed  \n",
       "3    steamed     mercedes      sever  recommended    brews  \n",
       "4      juicy     provided     humble     shawarma  dinning  \n",
       "5   frazzled       second        def       dashed   wanted  \n",
       "6  recommend      service      would         back   months  \n",
       "7    noticed        chair  selection        metal     even  \n",
       "8      group         girl    sashimi      couches    wraps  \n",
       "9   japanese        vowed       rabe   indicating    takes  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "### TOP WORDS (RANKING WEIGHTS ORDER)\n",
    "weight_words_topwords\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>chair</td>\n",
       "      <td>couches</td>\n",
       "      <td>fen</td>\n",
       "      <td>great</td>\n",
       "      <td>however</td>\n",
       "      <td>men</td>\n",
       "      <td>months</td>\n",
       "      <td>needed</td>\n",
       "      <td>recommended</td>\n",
       "      <td>setting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>chair</td>\n",
       "      <td>def</td>\n",
       "      <td>finding</td>\n",
       "      <td>great</td>\n",
       "      <td>perfect</td>\n",
       "      <td>recommended</td>\n",
       "      <td>retro</td>\n",
       "      <td>stevenonmill</td>\n",
       "      <td>text</td>\n",
       "      <td>told</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>back</td>\n",
       "      <td>batter</td>\n",
       "      <td>couches</td>\n",
       "      <td>great</td>\n",
       "      <td>ignored</td>\n",
       "      <td>months</td>\n",
       "      <td>needed</td>\n",
       "      <td>recommended</td>\n",
       "      <td>service</td>\n",
       "      <td>wait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>batter</td>\n",
       "      <td>brecksville</td>\n",
       "      <td>brews</td>\n",
       "      <td>chair</td>\n",
       "      <td>couches</td>\n",
       "      <td>cut</td>\n",
       "      <td>mercedes</td>\n",
       "      <td>recommended</td>\n",
       "      <td>sever</td>\n",
       "      <td>steamed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>breakfast</td>\n",
       "      <td>dinning</td>\n",
       "      <td>gab</td>\n",
       "      <td>humble</td>\n",
       "      <td>juicy</td>\n",
       "      <td>massieve</td>\n",
       "      <td>provided</td>\n",
       "      <td>realize</td>\n",
       "      <td>shawarma</td>\n",
       "      <td>shops</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>dashed</td>\n",
       "      <td>def</td>\n",
       "      <td>deserves</td>\n",
       "      <td>estate</td>\n",
       "      <td>frazzled</td>\n",
       "      <td>great</td>\n",
       "      <td>hair</td>\n",
       "      <td>imdb</td>\n",
       "      <td>second</td>\n",
       "      <td>wanted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>back</td>\n",
       "      <td>frock</td>\n",
       "      <td>great</td>\n",
       "      <td>mom</td>\n",
       "      <td>months</td>\n",
       "      <td>recommend</td>\n",
       "      <td>service</td>\n",
       "      <td>texting</td>\n",
       "      <td>time</td>\n",
       "      <td>would</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>chair</td>\n",
       "      <td>def</td>\n",
       "      <td>even</td>\n",
       "      <td>metal</td>\n",
       "      <td>months</td>\n",
       "      <td>noticed</td>\n",
       "      <td>selection</td>\n",
       "      <td>specific</td>\n",
       "      <td>time</td>\n",
       "      <td>would</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>chair</td>\n",
       "      <td>couches</td>\n",
       "      <td>girl</td>\n",
       "      <td>group</td>\n",
       "      <td>hey</td>\n",
       "      <td>recommended</td>\n",
       "      <td>sashimi</td>\n",
       "      <td>son</td>\n",
       "      <td>witnessed</td>\n",
       "      <td>wraps</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>25</td>\n",
       "      <td>alfresco</td>\n",
       "      <td>boy</td>\n",
       "      <td>indicating</td>\n",
       "      <td>japanese</td>\n",
       "      <td>mouth</td>\n",
       "      <td>rabe</td>\n",
       "      <td>somewhat</td>\n",
       "      <td>takes</td>\n",
       "      <td>vowed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cluster_lda    W_lda_0      W_lda_1   W_lda_2     W_lda_3   W_lda_4  \\\n",
       "0           0      chair      couches       fen       great   however   \n",
       "1           1      chair          def   finding       great   perfect   \n",
       "2           2       back       batter   couches       great   ignored   \n",
       "3           3     batter  brecksville     brews       chair   couches   \n",
       "4           4  breakfast      dinning       gab      humble     juicy   \n",
       "5           5     dashed          def  deserves      estate  frazzled   \n",
       "6           6       back        frock     great         mom    months   \n",
       "7           7      chair          def      even       metal    months   \n",
       "8           8      chair      couches      girl       group       hey   \n",
       "9           9         25     alfresco       boy  indicating  japanese   \n",
       "\n",
       "       W_lda_5    W_lda_6       W_lda_7      W_lda_8  W_lda_9  \n",
       "0          men     months        needed  recommended  setting  \n",
       "1  recommended      retro  stevenonmill         text     told  \n",
       "2       months     needed   recommended      service     wait  \n",
       "3          cut   mercedes   recommended        sever  steamed  \n",
       "4     massieve   provided       realize     shawarma    shops  \n",
       "5        great       hair          imdb       second   wanted  \n",
       "6    recommend    service       texting         time    would  \n",
       "7      noticed  selection      specific         time    would  \n",
       "8  recommended    sashimi           son    witnessed    wraps  \n",
       "9        mouth       rabe      somewhat        takes    vowed  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "### TOP WORDS (ALPHABETICAL ORDER)\n",
    "dataframe_topwords\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}