news-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os, sys, re, json, itertools, operator\n",
    "import pandas as pd, numpy as np, scipy as sp\n",
    "from joblib import Memory\n",
    "from bson import json_util\n",
    "    \n",
    "%load_ext cythonmagic       # '%' = magic function in ipython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_directory = os.path.join('data')\n",
    "output_directory = os.path.join(os.path.expanduser(\"~\"), 'work', 'stockmeme', 'news_analysis')\n",
    "\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)\n",
    "\n",
    "joblib_cache = Memory(cachedir=os.path.join(output_directory, 'joblib'), verbose=0)\n",
    "\n",
    "pd.set_option('notebook_repr_html', True)\n",
    "pd.set_option('precision', 4)\n",
    "pd.set_option('max_columns', 500)\n",
    "pd.set_option('max_rows', 50)\n",
    "pd.set_option('max_colwidth', 500)\n",
    "pd.set_option('column_space', 100)\n",
    "pd.set_option('use_inf_as_null', True)\n",
    "\n",
    "#np.set_printoptions(linewidth=200, precision=15, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>URL</th>\n",
       "      <th>created_at</th>\n",
       "      <th>feed_URL</th>\n",
       "      <th>insertion_date</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> MSQ4UR6JIJV801</td>\n",
       "      <td>          http://www.bloomberg.com/news/2013-09-06/bank-of-america-settles-merrill-sex-discrimination-case.html</td>\n",
       "      <td> 2013-09-07 04:01:01+00:00</td>\n",
       "      <td> http://www.bloomberg.com/news/industries/</td>\n",
       "      <td> 2013-09-07 14:26:22.006000+00:00</td>\n",
       "      <td> [Lawyers representing female financial advisers and licensed\\ntrainees filed a request for approval of the settlement\\nyesterday in federal court in Brooklyn , New York . The advisers\\nalleged in the lawsuit that Merrill discriminated against women\\nin those positions in compensation and business opportunities., “This settlement helps ensure that Merrill Lynch is a\\nplace where women can thrive and be successful,” Cara E.\\nGreene, a lawyer for the plaintiffs, said in a statement.\\n“Hopefully...</td>\n",
       "      <td>                                                                         Bank of America Settles Merrill Sex Discrimination Case Bank of America Corp. will pay $39\\nmillion to settle a discrimination lawsuit brought on behalf of\\nwomen employees at its Merrill Lynch unit, according to court\\nfilings.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> MSQ6GX0D9L3501</td>\n",
       "      <td>   http://www.bloomberg.com/news/2013-09-07/fed-seen-set-to-taper-qe-even-as-payroll-gains-trail-forecasts.html</td>\n",
       "      <td> 2013-09-07 04:00:01+00:00</td>\n",
       "      <td>    http://www.bloomberg.com/news/markets/</td>\n",
       "      <td> 2013-09-07 14:26:08.001000+00:00</td>\n",
       "      <td> [Payrolls rose by 169,000 last month, a Labor Department\\nreport showed yesterday in Washington , and the jobless rate fell\\nto 7.3 percent as people left the workforce. Revisions to prior\\nreports subtracted a total of 74,000 jobs to payrolls in the\\nprevious two months, while hours worked and earnings rose., The jobs report is the last one Fed policy makers will see\\nbefore their Sept. 17-18 meeting, when they resume a debate on\\nwhen to pare $85 billion in monthly bond purchases. The incr...</td>\n",
       "      <td>                                                                                                                                                                                                                                               Fed Seen Set to Taper QE Even as Payroll Gains Trail Forecasts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> MSPBA90D9L3501</td>\n",
       "      <td> http://www.bloomberg.com/news/2013-09-07/germany-s-bonds-decline-as-ecb-s-draghi-fails-to-reverse-selloff.html</td>\n",
       "      <td> 2013-09-07 06:00:00+00:00</td>\n",
       "      <td>    http://www.bloomberg.com/news/markets/</td>\n",
       "      <td> 2013-09-07 06:21:52.930000+00:00</td>\n",
       "      <td> [Austrian, French and Dutch (GNTH10YR) 10-year yields all climbed to\\nthe highest in more than a year as most government bond markets\\nacross the euro region tumbled along with Treasuries. Draghi\\nsaid the ECB “expects the key ECB interest rates to remain at\\npresent or lower levels for an extended period of time,” at a\\npress conference in Frankfurt on Sept. 5. Reports showed\\nmanufacturing in the euro area and China expanded last month., “We’ve seen the European Central Bank ’s not willing...</td>\n",
       "      <td>                      Germany’s Bonds Decline as ECB’s Draghi Fails to Reverse Selloff Germany’s bonds fell this week,\\npushing 10-year yields to a 17-month high, as European Central\\nBank President Mario Draghi failed to convince investors\\ninterest rates will remain low amid global economic growth.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> MSPGO21A1I4H01</td>\n",
       "      <td>   http://www.bloomberg.com/news/2013-09-07/gilts-decline-in-longest-streak-for-six-years-on-recovery-bets.html</td>\n",
       "      <td> 2013-09-07 06:00:00+00:00</td>\n",
       "      <td>    http://www.bloomberg.com/news/markets/</td>\n",
       "      <td> 2013-09-07 06:21:51.640000+00:00</td>\n",
       "      <td> [Sterling rose for the first time in three weeks against the\\ndollar as a report showed the U.S. economy added fewer jobs in\\nAugust than economists projected. It also advanced against the\\neuro. Ten-year yields climbed above 3 percent for the first time\\nsince July 2011 after the Bank of England’s Monetary Policy\\nCommittee left its stimulus program unchanged and kept its\\nbenchmark rate at a record low., “An increase in yields appears to be justified by the\\nrecent domestic data and events...</td>\n",
       "      <td>                                 Gilts Decline in Longest Streak for Six Years on Recovery Bets Gilts declined for a seventh week,\\nthe longest streak in more than six years, as reports showed\\nU.K. manufacturing and services industries expanded last month,\\nadding to evidence of accelerating growth.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> MSPYP46S973201</td>\n",
       "      <td>    http://www.bloomberg.com/news/2013-09-07/dollar-rises-versus-euro-yen-on-taper-bets-amid-syria-tension.html</td>\n",
       "      <td> 2013-09-07 04:00:11+00:00</td>\n",
       "      <td>    http://www.bloomberg.com/news/markets/</td>\n",
       "      <td> 2013-09-07 04:26:23.808000+00:00</td>\n",
       "      <td> [The greenback reached a eight-week high versus the 17-nation currency as the prospect for U.S. intervention in Syria increased demand for safer assets. The Australian and New\\nZealand dollars gained as China ’s service industry expanded last\\nmonth. The U.S. currency pared gains versus the euro and the yen\\nyesterday after U.S. employers added fewer workers than forecast\\nin August, even as surveys showed economists’ retained Fed taper\\nprojections. U.S. retail sales growth quickened last m...</td>\n",
       "      <td>                                                                                    Dollar Rises Versus Euro, Yen on Taper Bets Amid Syria Tension The dollar gained versus the euro\\nand yen on speculation the Federal Reserve will trim the pace of\\nits $85 billion in monthly bond purchases this month.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td> MSQB4E6JIJVD01</td>\n",
       "      <td>             http://www.bloomberg.com/news/2013-09-07/apple-said-to-be-close-to-clinching-carrier-holdouts.html</td>\n",
       "      <td> 2013-09-07 00:09:50+00:00</td>\n",
       "      <td> http://www.bloomberg.com/news/industries/</td>\n",
       "      <td> 2013-09-07 00:29:55.088000+00:00</td>\n",
       "      <td> [On the eve of an event on Sept. 10 to unveil new iPhones,\\nApple and China Mobile are near a deal for the world’s largest\\nwireless carrier to offer the device, said a person with\\nknowledge of the matter, who asked not to be identified because\\nthe discussions are private. Apple is also close to an agreement\\nto distribute its handset through Japan ’s largest mobile\\ncarrier, NTT DoCoMo, people with knowledge of the situation have\\nsaid., Any final deals will bust loose at least 800 millio...</td>\n",
       "      <td>                                          Apple Said to Be Close to Clinching Carrier Holdouts Apple Inc. is on the verge of\\nclinching agreements with two of the biggest holdouts to selling\\nits iPhone, China Mobile Ltd. and NTT DoCoMo Inc., opening a\\nroute to hundreds of millions of new customers.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td> MSP6SQ6JTSE801</td>\n",
       "      <td>   http://www.bloomberg.com/news/2013-09-06/everbright-errant-china-trades-lead-to-85-million-monthly-loss.html</td>\n",
       "      <td> 2013-09-06 16:00:00+00:00</td>\n",
       "      <td>    http://www.bloomberg.com/news/finance/</td>\n",
       "      <td> 2013-09-07 00:07:52.110000+00:00</td>\n",
       "      <td> [The country’s seventh-largest brokerage by market value\\nincurred the unconsolidated loss, which doesn’t include\\ncontributions from its subsidiaries, after disposing of assets\\nfollowing the Aug. 16 error, Everbright said in a statement to\\nthe Shanghai Stock Exchange yesterday. The figure is based on\\npreliminary data and hasn’t been audited, the company said., The loss adds to woes at state-controlled Everbright, which\\nhas been ordered by the China Securities Regulatory Commission\\nto p...</td>\n",
       "      <td>           Everbright Errant China Trades Lead to $85 Million Monthly Loss Everbright Securities Co. posted an\\nunconsolidated net loss of 523 million yuan ($85 million) last\\nmonth after $3.8 billion in erroneous trading orders that roiled\\nChina’s equity market and drew a record regulatory penalty.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td> MSQ94Q6JTSEE01</td>\n",
       "      <td>          http://www.bloomberg.com/news/2013-09-07/florida-ferris-wheel-deal-prompts-real-estate-sale-suit.html</td>\n",
       "      <td> 2013-09-07 00:02:52+00:00</td>\n",
       "      <td> http://www.bloomberg.com/news/industries/</td>\n",
       "      <td> 2013-09-07 00:07:44.758000+00:00</td>\n",
       "      <td> [Lawyers for Great Wheel Beteliligungs GMBH &amp; Co. KG, formed\\nto build a giant Ferris wheel in Berlin, contends the developer\\nof the defunct U.S. attraction wants to improperly withhold more\\nthan $1.2 million from the sale of the site. The German company,\\nwhich says it invested $63 million in the Orlando, Florida,\\nproject, wants Delaware Chancery Court Judge Travis Laster to\\nappoint a receiver to monitor the sale., The sale of the Great Orlando Wheel Corp. ’s assets “would\\nresult in a ...</td>\n",
       "      <td>                   Florida Ferris Wheel Deal Prompts Real Estate Sale Suit An investor in a failed bid to build\\na 400-foot-high Ferris wheel in Florida is asking a judge to\\nappoint an independent monitor to oversee the sale of the\\nattraction’s proposed site, according to an unsealed court\\nfiling.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td> MOX13T6K50XT01</td>\n",
       "      <td>    http://www.bloomberg.com/news/2013-09-06/smithfield-receives-u-s-regulator-approval-for-shuanghui-deal.html</td>\n",
       "      <td> 2013-09-06 23:37:11+00:00</td>\n",
       "      <td>    http://www.bloomberg.com/news/markets/</td>\n",
       "      <td> 2013-09-06 23:39:58.747000+00:00</td>\n",
       "      <td> [“This transaction will create a leading global animal\\nprotein enterprise,” Zhijun Yang, Chief Executive Officer of\\nShuanghui International, said in a joint statement from both\\ncompanies released yesterday., The Committee on Foreign Investment in the U.S., or CFIUS,\\napproved the transaction and it will be voted on by Smithfield\\nshareholders at the company’s annual meeting Sept. 24. The\\ngovernment of Ukraine also approved the deal, according to the\\nstatement., Shuanghui, based in Hong ...</td>\n",
       "      <td> Smithfield Receives U.S. Approval for Biggest Chinese Takeover Smithfield Foods Inc., the world’s\\nlargest hog and pork producer, said U.S. regulators will allow\\nthe company to be bought by China’s Shuanghui International\\nHoldings Ltd. in what would be the biggest Chinese purchase of a\\nU.S. firm.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td> MSQ6KQ6JIJUY01</td>\n",
       "      <td>          http://www.bloomberg.com/news/2013-09-06/malkins-reject-empire-state-building-s-buyout-proposals.html</td>\n",
       "      <td> 2013-09-06 22:46:52+00:00</td>\n",
       "      <td> http://www.bloomberg.com/news/industries/</td>\n",
       "      <td> 2013-09-06 23:07:03.555000+00:00</td>\n",
       "      <td> [The company reviewed the proposals to purchase control of\\nthe Empire State Building or its operating lease positions and\\ndetermined that an initial public offering is in investors’ best\\ninterest, according to a regulatory filing today. Lazard Ltd.\\nacted as the Malkins’ independent adviser., Bidders including Rubin Schron ’s Cammeby’s International\\nand Joseph Sitt ’s Thor Equities LLC made unsolicited offers for\\nthe Empire State Building as an alternative to the planned IPO.\\nMalkin Ho...</td>\n",
       "      <td>                                            Malkins Reject Empire State Building’s Buyout Proposals Malkin Holdings LLC, the supervisor\\nof New York’s Empire State Building, is proceeding with a plan\\nto form a publicly traded real estate investment trust after\\nreceiving buyout offers for the tower.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               ID  \\\n",
       "0  MSQ4UR6JIJV801   \n",
       "1  MSQ6GX0D9L3501   \n",
       "2  MSPBA90D9L3501   \n",
       "3  MSPGO21A1I4H01   \n",
       "4  MSPYP46S973201   \n",
       "5  MSQB4E6JIJVD01   \n",
       "6  MSP6SQ6JTSE801   \n",
       "7  MSQ94Q6JTSEE01   \n",
       "8  MOX13T6K50XT01   \n",
       "9  MSQ6KQ6JIJUY01   \n",
       "\n",
       "                                                                                                              URL  \\\n",
       "0           http://www.bloomberg.com/news/2013-09-06/bank-of-america-settles-merrill-sex-discrimination-case.html   \n",
       "1    http://www.bloomberg.com/news/2013-09-07/fed-seen-set-to-taper-qe-even-as-payroll-gains-trail-forecasts.html   \n",
       "2  http://www.bloomberg.com/news/2013-09-07/germany-s-bonds-decline-as-ecb-s-draghi-fails-to-reverse-selloff.html   \n",
       "3    http://www.bloomberg.com/news/2013-09-07/gilts-decline-in-longest-streak-for-six-years-on-recovery-bets.html   \n",
       "4     http://www.bloomberg.com/news/2013-09-07/dollar-rises-versus-euro-yen-on-taper-bets-amid-syria-tension.html   \n",
       "5              http://www.bloomberg.com/news/2013-09-07/apple-said-to-be-close-to-clinching-carrier-holdouts.html   \n",
       "6    http://www.bloomberg.com/news/2013-09-06/everbright-errant-china-trades-lead-to-85-million-monthly-loss.html   \n",
       "7           http://www.bloomberg.com/news/2013-09-07/florida-ferris-wheel-deal-prompts-real-estate-sale-suit.html   \n",
       "8     http://www.bloomberg.com/news/2013-09-06/smithfield-receives-u-s-regulator-approval-for-shuanghui-deal.html   \n",
       "9           http://www.bloomberg.com/news/2013-09-06/malkins-reject-empire-state-building-s-buyout-proposals.html   \n",
       "\n",
       "                  created_at                                   feed_URL  \\\n",
       "0  2013-09-07 04:01:01+00:00  http://www.bloomberg.com/news/industries/   \n",
       "1  2013-09-07 04:00:01+00:00     http://www.bloomberg.com/news/markets/   \n",
       "2  2013-09-07 06:00:00+00:00     http://www.bloomberg.com/news/markets/   \n",
       "3  2013-09-07 06:00:00+00:00     http://www.bloomberg.com/news/markets/   \n",
       "4  2013-09-07 04:00:11+00:00     http://www.bloomberg.com/news/markets/   \n",
       "5  2013-09-07 00:09:50+00:00  http://www.bloomberg.com/news/industries/   \n",
       "6  2013-09-06 16:00:00+00:00     http://www.bloomberg.com/news/finance/   \n",
       "7  2013-09-07 00:02:52+00:00  http://www.bloomberg.com/news/industries/   \n",
       "8  2013-09-06 23:37:11+00:00     http://www.bloomberg.com/news/markets/   \n",
       "9  2013-09-06 22:46:52+00:00  http://www.bloomberg.com/news/industries/   \n",
       "\n",
       "                     insertion_date  \\\n",
       "0  2013-09-07 14:26:22.006000+00:00   \n",
       "1  2013-09-07 14:26:08.001000+00:00   \n",
       "2  2013-09-07 06:21:52.930000+00:00   \n",
       "3  2013-09-07 06:21:51.640000+00:00   \n",
       "4  2013-09-07 04:26:23.808000+00:00   \n",
       "5  2013-09-07 00:29:55.088000+00:00   \n",
       "6  2013-09-07 00:07:52.110000+00:00   \n",
       "7  2013-09-07 00:07:44.758000+00:00   \n",
       "8  2013-09-06 23:39:58.747000+00:00   \n",
       "9  2013-09-06 23:07:03.555000+00:00   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  text  \\\n",
       "0  [Lawyers representing female financial advisers and licensed\\ntrainees filed a request for approval of the settlement\\nyesterday in federal court in Brooklyn , New York . The advisers\\nalleged in the lawsuit that Merrill discriminated against women\\nin those positions in compensation and business opportunities., “This settlement helps ensure that Merrill Lynch is a\\nplace where women can thrive and be successful,” Cara E.\\nGreene, a lawyer for the plaintiffs, said in a statement.\\n“Hopefully...   \n",
       "1  [Payrolls rose by 169,000 last month, a Labor Department\\nreport showed yesterday in Washington , and the jobless rate fell\\nto 7.3 percent as people left the workforce. Revisions to prior\\nreports subtracted a total of 74,000 jobs to payrolls in the\\nprevious two months, while hours worked and earnings rose., The jobs report is the last one Fed policy makers will see\\nbefore their Sept. 17-18 meeting, when they resume a debate on\\nwhen to pare $85 billion in monthly bond purchases. The incr...   \n",
       "2  [Austrian, French and Dutch (GNTH10YR) 10-year yields all climbed to\\nthe highest in more than a year as most government bond markets\\nacross the euro region tumbled along with Treasuries. Draghi\\nsaid the ECB “expects the key ECB interest rates to remain at\\npresent or lower levels for an extended period of time,” at a\\npress conference in Frankfurt on Sept. 5. Reports showed\\nmanufacturing in the euro area and China expanded last month., “We’ve seen the European Central Bank ’s not willing...   \n",
       "3  [Sterling rose for the first time in three weeks against the\\ndollar as a report showed the U.S. economy added fewer jobs in\\nAugust than economists projected. It also advanced against the\\neuro. Ten-year yields climbed above 3 percent for the first time\\nsince July 2011 after the Bank of England’s Monetary Policy\\nCommittee left its stimulus program unchanged and kept its\\nbenchmark rate at a record low., “An increase in yields appears to be justified by the\\nrecent domestic data and events...   \n",
       "4  [The greenback reached a eight-week high versus the 17-nation currency as the prospect for U.S. intervention in Syria increased demand for safer assets. The Australian and New\\nZealand dollars gained as China ’s service industry expanded last\\nmonth. The U.S. currency pared gains versus the euro and the yen\\nyesterday after U.S. employers added fewer workers than forecast\\nin August, even as surveys showed economists’ retained Fed taper\\nprojections. U.S. retail sales growth quickened last m...   \n",
       "5  [On the eve of an event on Sept. 10 to unveil new iPhones,\\nApple and China Mobile are near a deal for the world’s largest\\nwireless carrier to offer the device, said a person with\\nknowledge of the matter, who asked not to be identified because\\nthe discussions are private. Apple is also close to an agreement\\nto distribute its handset through Japan ’s largest mobile\\ncarrier, NTT DoCoMo, people with knowledge of the situation have\\nsaid., Any final deals will bust loose at least 800 millio...   \n",
       "6  [The country’s seventh-largest brokerage by market value\\nincurred the unconsolidated loss, which doesn’t include\\ncontributions from its subsidiaries, after disposing of assets\\nfollowing the Aug. 16 error, Everbright said in a statement to\\nthe Shanghai Stock Exchange yesterday. The figure is based on\\npreliminary data and hasn’t been audited, the company said., The loss adds to woes at state-controlled Everbright, which\\nhas been ordered by the China Securities Regulatory Commission\\nto p...   \n",
       "7  [Lawyers for Great Wheel Beteliligungs GMBH & Co. KG, formed\\nto build a giant Ferris wheel in Berlin, contends the developer\\nof the defunct U.S. attraction wants to improperly withhold more\\nthan $1.2 million from the sale of the site. The German company,\\nwhich says it invested $63 million in the Orlando, Florida,\\nproject, wants Delaware Chancery Court Judge Travis Laster to\\nappoint a receiver to monitor the sale., The sale of the Great Orlando Wheel Corp. ’s assets “would\\nresult in a ...   \n",
       "8  [“This transaction will create a leading global animal\\nprotein enterprise,” Zhijun Yang, Chief Executive Officer of\\nShuanghui International, said in a joint statement from both\\ncompanies released yesterday., The Committee on Foreign Investment in the U.S., or CFIUS,\\napproved the transaction and it will be voted on by Smithfield\\nshareholders at the company’s annual meeting Sept. 24. The\\ngovernment of Ukraine also approved the deal, according to the\\nstatement., Shuanghui, based in Hong ...   \n",
       "9  [The company reviewed the proposals to purchase control of\\nthe Empire State Building or its operating lease positions and\\ndetermined that an initial public offering is in investors’ best\\ninterest, according to a regulatory filing today. Lazard Ltd.\\nacted as the Malkins’ independent adviser., Bidders including Rubin Schron ’s Cammeby’s International\\nand Joseph Sitt ’s Thor Equities LLC made unsolicited offers for\\nthe Empire State Building as an alternative to the planned IPO.\\nMalkin Ho...   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                          title  \n",
       "0                                                                          Bank of America Settles Merrill Sex Discrimination Case Bank of America Corp. will pay $39\\nmillion to settle a discrimination lawsuit brought on behalf of\\nwomen employees at its Merrill Lynch unit, according to court\\nfilings.  \n",
       "1                                                                                                                                                                                                                                                Fed Seen Set to Taper QE Even as Payroll Gains Trail Forecasts  \n",
       "2                       Germany’s Bonds Decline as ECB’s Draghi Fails to Reverse Selloff Germany’s bonds fell this week,\\npushing 10-year yields to a 17-month high, as European Central\\nBank President Mario Draghi failed to convince investors\\ninterest rates will remain low amid global economic growth.  \n",
       "3                                  Gilts Decline in Longest Streak for Six Years on Recovery Bets Gilts declined for a seventh week,\\nthe longest streak in more than six years, as reports showed\\nU.K. manufacturing and services industries expanded last month,\\nadding to evidence of accelerating growth.  \n",
       "4                                                                                     Dollar Rises Versus Euro, Yen on Taper Bets Amid Syria Tension The dollar gained versus the euro\\nand yen on speculation the Federal Reserve will trim the pace of\\nits $85 billion in monthly bond purchases this month.  \n",
       "5                                           Apple Said to Be Close to Clinching Carrier Holdouts Apple Inc. is on the verge of\\nclinching agreements with two of the biggest holdouts to selling\\nits iPhone, China Mobile Ltd. and NTT DoCoMo Inc., opening a\\nroute to hundreds of millions of new customers.  \n",
       "6            Everbright Errant China Trades Lead to $85 Million Monthly Loss Everbright Securities Co. posted an\\nunconsolidated net loss of 523 million yuan ($85 million) last\\nmonth after $3.8 billion in erroneous trading orders that roiled\\nChina’s equity market and drew a record regulatory penalty.  \n",
       "7                    Florida Ferris Wheel Deal Prompts Real Estate Sale Suit An investor in a failed bid to build\\na 400-foot-high Ferris wheel in Florida is asking a judge to\\nappoint an independent monitor to oversee the sale of the\\nattraction’s proposed site, according to an unsealed court\\nfiling.  \n",
       "8  Smithfield Receives U.S. Approval for Biggest Chinese Takeover Smithfield Foods Inc., the world’s\\nlargest hog and pork producer, said U.S. regulators will allow\\nthe company to be bought by China’s Shuanghui International\\nHoldings Ltd. in what would be the biggest Chinese purchase of a\\nU.S. firm.  \n",
       "9                                             Malkins Reject Empire State Building’s Buyout Proposals Malkin Holdings LLC, the supervisor\\nof New York’s Empire State Building, is proceeding with a plan\\nto form a publicly traded real estate investment trust after\\nreceiving buyout offers for the tower.  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open( os.path.join(data_directory, 'bloomberg.json') ) as f:\n",
    "    bloomberg = json_util.loads(f.read())   #bloomberg = large dictionary (b/c json format)\n",
    "\n",
    "df_bloomberg = pd.DataFrame(bloomberg)      #pandas can take json data & build data pandas \n",
    "df_bloomberg = df_bloomberg.drop('_id', axis=1); #drop 'name of colmumn'; axis = 1 specifies that '_id\" is a column not a row (1 = row)\n",
    "df_bloomberg = df_bloomberg.sort('insertion_date', ascending=False)\n",
    "df_bloomberg = df_bloomberg.reset_index().drop('index', axis=1)\n",
    "## drop, sort etc are pandas methods -- hence used on pandas obj \n",
    "df_bloomberg.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>URL</th>\n",
       "      <th>created_at</th>\n",
       "      <th>feed_URL</th>\n",
       "      <th>insertion_date</th>\n",
       "      <th>tags</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> http://www.reuters.com/article/2013/09/07/film-toronto-festival-prisoners-idUSL2N0H308X20130907</td>\n",
       "      <td> http://www.reuters.com/article/2013/09/07/film-toronto-festival-prisoners-idUSL2N0H308X20130907</td>\n",
       "      <td> 2013-09-07 14:09:11+00:00</td>\n",
       "      <td>                          http://feeds.reuters.com/news/usmarkets</td>\n",
       "      <td> 2013-09-07 14:35:16.164000+00:00</td>\n",
       "      <td>                            [{u'term': u'marketsNews', u'scheme': None, u'label': None}]</td>\n",
       "      <td> [TORONTO, Sept 7  - Forget for a minute the Hugh\\nJackman of Broadway musicals  and \"Les Miserables\" and \"X-Men\"\\nmovies, and envision the Australian actor as a Pennsylvania\\nsurvivalist and desperate father who takes justice into his own\\nhands., As a carpenter without enough work who stockpiles supplies\\nand doesn't trust government, he is the seething vigilante who\\ndrives the dark thriller \"Prisoners,\" one of the most\\ntalked-about films at the Toronto International Film Festival\\nand th...</td>\n",
       "      <td> 'Prisoners' captures rage and despair of a vulnerable America at Toronto film fest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>            http://www.reuters.com/article/2013/09/07/cernobbio-saccomanni-idUSL5N0H30A920130907</td>\n",
       "      <td>            http://www.reuters.com/article/2013/09/07/cernobbio-saccomanni-idUSL5N0H30A920130907</td>\n",
       "      <td> 2013-09-07 13:50:56+00:00</td>\n",
       "      <td>                       http://feeds.reuters.com/reuters/bondsNews</td>\n",
       "      <td> 2013-09-07 14:04:41.071000+00:00</td>\n",
       "      <td>                              [{u'term': u'bondsNews', u'scheme': None, u'label': None}]</td>\n",
       "      <td>                                                                                                                                                                                                                                                                                                                                                 [* Warns of \"disastrous loss of credibility\" if deficit\\ntarget missed, * Berlusconi party pledges \"responsibility\" ahead of key\\nSenate vote, By Giancarlo Navach]</td>\n",
       "      <td>                           UPDATE 1-Minister says Italy will dodge political crisis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>           http://www.reuters.com/article/2013/09/07/telecomitalia-sawiris-idUSL5N0H30CG20130907</td>\n",
       "      <td>           http://www.reuters.com/article/2013/09/07/telecomitalia-sawiris-idUSL5N0H30CG20130907</td>\n",
       "      <td> 2013-09-07 13:11:33+00:00</td>\n",
       "      <td>       http://feeds.reuters.com/reuters/cyclicalconsumergoodsNews</td>\n",
       "      <td> 2013-09-07 13:22:29.240000+00:00</td>\n",
       "      <td>            [{u'term': u'cyclicalConsumerGoodsSector', u'scheme': None, u'label': None}]</td>\n",
       "      <td>                                  [By Paola Arosio, MILAN, Sept 7 (Reuters) - Egyptian telecoms tycoon Naguib\\nSawiris said on Saturday he was still interested in taking a\\nstake in Telecom Italia but might be discouraged if\\nthe Italian government was opposed., In an emailed statement to Reuters, Sawiris said he was\\nconsidering a \"potential investment in Telecom Italia\" but might\\nnot submit a proposal \"in view of (Italy's)... alleged\\npreference for Telefonica reported in the Italian press.\"]</td>\n",
       "      <td>                     UPDATE 1-Sawiris eyes Telecom Italia, but concerned about Rome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>        http://www.reuters.com/article/2013/09/07/usa-philadelphia-schools-idUSL2N0GO1CY20130907</td>\n",
       "      <td>        http://www.reuters.com/article/2013/09/07/usa-philadelphia-schools-idUSL2N0GO1CY20130907</td>\n",
       "      <td> 2013-09-07 13:00:00+00:00</td>\n",
       "      <td>                       http://feeds.reuters.com/reuters/bondsNews</td>\n",
       "      <td> 2013-09-07 13:20:11.633000+00:00</td>\n",
       "      <td>                              [{u'term': u'bondsNews', u'scheme': None, u'label': None}]</td>\n",
       "      <td>                                                                                                                                                                                                                                                                                                                                                                                                            [* City pensions compete with schools for extra cash, * Plans to sell shuttered schools, By Hilary Russ]</td>\n",
       "      <td>                    Fiscal crisis looms for Philadelphia schools as students return</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>             http://www.reuters.com/article/2013/09/07/china-politics-ring-idUSL4N0H306B20130907</td>\n",
       "      <td>             http://www.reuters.com/article/2013/09/07/china-politics-ring-idUSL4N0H306B20130907</td>\n",
       "      <td> 2013-09-07 11:09:50+00:00</td>\n",
       "      <td>       http://feeds.reuters.com/reuters/cyclicalconsumergoodsNews</td>\n",
       "      <td> 2013-09-07 12:53:21.464000+00:00</td>\n",
       "      <td>            [{u'term': u'cyclicalConsumerGoodsSector', u'scheme': None, u'label': None}]</td>\n",
       "      <td> [The official Xinhua news agency put out a one-line report\\nstating: \"Talk on the Internet about Xi Jinping wearing a\\nwedding ring at the G20 summit is fake information.\" It provided\\nno further explanation., The denial comes as China embarks on yet another crackdown\\non what it terms \"online rumours\", as the control-obsessed\\ngovernment tries once again to rein in social media., Xinhua's report appeared to stem from a lingering shot\\ncarried on state television earlier in the week of Xi's ...</td>\n",
       "      <td>                              Chinese left confused by bizarre missive on Xi's ring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>              http://www.reuters.com/article/2013/09/07/australia-election-idUSL4N0H307Q20130907</td>\n",
       "      <td>              http://www.reuters.com/article/2013/09/07/australia-election-idUSL4N0H307Q20130907</td>\n",
       "      <td> 2013-09-07 12:34:32+00:00</td>\n",
       "      <td>                          http://feeds.reuters.com/news/usmarkets</td>\n",
       "      <td> 2013-09-07 12:53:04.493000+00:00</td>\n",
       "      <td>                            [{u'term': u'marketsNews', u'scheme': None, u'label': None}]</td>\n",
       "      <td>                                                                                                                                                                                                                                                                                                                                                                                   [* Australian conservatives win power, * Defeated Rudd quits as Labor leader, * Voters punish Labor party for political disunity]</td>\n",
       "      <td>  RPT-UPDATE 5-Conservative leader Abbott sweeps into power in Australian elections</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>            http://www.reuters.com/article/2013/09/07/india-finmin-imports-idUSL4N0H306M20130907</td>\n",
       "      <td>            http://www.reuters.com/article/2013/09/07/india-finmin-imports-idUSL4N0H306M20130907</td>\n",
       "      <td> 2013-09-07 11:34:40+00:00</td>\n",
       "      <td>                  http://feeds.reuters.com/reuters/financialsNews</td>\n",
       "      <td> 2013-09-07 12:12:56.359000+00:00</td>\n",
       "      <td>                       [{u'term': u'financialsSector', u'scheme': None, u'label': None}]</td>\n",
       "      <td>                                                                                                                                                           [The government has already banned duty-free import of\\nflat-screen televisions from Aug. 26, adding to a package of\\nmeasures designed to prop up the rupee., \"I hope that all these measures taken together will have a\\nbeneficial impact on inflation,\" Chidambaram told lawmakers., He added that no decision has been taken to raise fuel\\nprices.]</td>\n",
       "      <td>                           India to unveil new measures to curb inessential imports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>           http://www.reuters.com/article/2013/09/07/us-weather-hurricanes-idUSBRE9860AY20130907</td>\n",
       "      <td>           http://www.reuters.com/article/2013/09/07/us-weather-hurricanes-idUSBRE9860AY20130907</td>\n",
       "      <td> 2013-09-07 12:04:04+00:00</td>\n",
       "      <td>                     http://feeds.reuters.com/reuters/environment</td>\n",
       "      <td> 2013-09-07 12:12:56.359000+00:00</td>\n",
       "      <td>                        [{u'term': u'environmentNews', u'scheme': None, u'label': None}]</td>\n",
       "      <td> [MIAMI  - The 2013 Atlantic hurricane season, which forecasters had predicted would be more active than normal, has turned out to be something of a dud so far as an unusual calm hangs over the tropics., As the season heads into the historic peak for activity, it may even enter the record books as marking the quietest start to any Atlantic hurricane season in decades., \"It certainly looks like pretty much of a forecast bust,\" said Jeff Masters, a hurricane expert and director of meteorology a...</td>\n",
       "      <td>                                 Atlantic hurricane season - a record-breaking dud?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>                http://www.reuters.com/article/2013/09/07/cernobbio-intesa-idUSL5N0H306K20130907</td>\n",
       "      <td>                http://www.reuters.com/article/2013/09/07/cernobbio-intesa-idUSL5N0H306K20130907</td>\n",
       "      <td> 2013-09-07 11:49:29+00:00</td>\n",
       "      <td> http://feeds.reuters.com/reuters/financialServicesrealEstateNews</td>\n",
       "      <td> 2013-09-07 12:12:37.609000+00:00</td>\n",
       "      <td> [{u'term': u'rbssFinancialServicesAndRealEstateNews', u'scheme': None, u'label': None}]</td>\n",
       "      <td>                                                                                                                                                                     [By Lisa Jucca and Paola Arosio, CERNOBBIO, Italy, Sept 7 (Reuters) - The chief executive of\\nIntesa Sanpaolo said on Saturday he expected a new\\nphase of mergers in the banking sector, both inside Italy and\\nacross Europe., Enrico Cucchiani also said his bank was not interested in\\nincreasing its exposure to Italy's banking system.]</td>\n",
       "      <td>                                INTERVIEW-Intesa CEO expects bank mergers in Europe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>              http://www.reuters.com/article/2013/09/07/weather-hurricanes-idUSL2N0H21LR20130907</td>\n",
       "      <td>              http://www.reuters.com/article/2013/09/07/weather-hurricanes-idUSL2N0H21LR20130907</td>\n",
       "      <td> 2013-09-07 12:00:00+00:00</td>\n",
       "      <td> http://feeds.reuters.com/reuters/financialServicesrealEstateNews</td>\n",
       "      <td> 2013-09-07 12:12:37.609000+00:00</td>\n",
       "      <td> [{u'term': u'rbssFinancialServicesAndRealEstateNews', u'scheme': None, u'label': None}]</td>\n",
       "      <td> [MIAMI, Sept 7  - The 2013 Atlantic hurricane\\nseason, which forecasters had predicted would be more active\\nthan normal, has turned out to be something of a dud so far as\\nan unusual calm hangs over the tropics., As the season heads into the historic peak for activity, it\\nmay even enter the record books as marking the quietest start to\\nany Atlantic hurricane season in decades., \"It certainly looks like pretty much of a forecast bust,\"\\nsaid Jeff Masters, a hurricane expert and director of...</td>\n",
       "      <td>                                 Atlantic hurricane season - a record-breaking dud?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                ID  \\\n",
       "0  http://www.reuters.com/article/2013/09/07/film-toronto-festival-prisoners-idUSL2N0H308X20130907   \n",
       "1             http://www.reuters.com/article/2013/09/07/cernobbio-saccomanni-idUSL5N0H30A920130907   \n",
       "2            http://www.reuters.com/article/2013/09/07/telecomitalia-sawiris-idUSL5N0H30CG20130907   \n",
       "3         http://www.reuters.com/article/2013/09/07/usa-philadelphia-schools-idUSL2N0GO1CY20130907   \n",
       "4              http://www.reuters.com/article/2013/09/07/china-politics-ring-idUSL4N0H306B20130907   \n",
       "5               http://www.reuters.com/article/2013/09/07/australia-election-idUSL4N0H307Q20130907   \n",
       "6             http://www.reuters.com/article/2013/09/07/india-finmin-imports-idUSL4N0H306M20130907   \n",
       "7            http://www.reuters.com/article/2013/09/07/us-weather-hurricanes-idUSBRE9860AY20130907   \n",
       "8                 http://www.reuters.com/article/2013/09/07/cernobbio-intesa-idUSL5N0H306K20130907   \n",
       "9               http://www.reuters.com/article/2013/09/07/weather-hurricanes-idUSL2N0H21LR20130907   \n",
       "\n",
       "                                                                                               URL  \\\n",
       "0  http://www.reuters.com/article/2013/09/07/film-toronto-festival-prisoners-idUSL2N0H308X20130907   \n",
       "1             http://www.reuters.com/article/2013/09/07/cernobbio-saccomanni-idUSL5N0H30A920130907   \n",
       "2            http://www.reuters.com/article/2013/09/07/telecomitalia-sawiris-idUSL5N0H30CG20130907   \n",
       "3         http://www.reuters.com/article/2013/09/07/usa-philadelphia-schools-idUSL2N0GO1CY20130907   \n",
       "4              http://www.reuters.com/article/2013/09/07/china-politics-ring-idUSL4N0H306B20130907   \n",
       "5               http://www.reuters.com/article/2013/09/07/australia-election-idUSL4N0H307Q20130907   \n",
       "6             http://www.reuters.com/article/2013/09/07/india-finmin-imports-idUSL4N0H306M20130907   \n",
       "7            http://www.reuters.com/article/2013/09/07/us-weather-hurricanes-idUSBRE9860AY20130907   \n",
       "8                 http://www.reuters.com/article/2013/09/07/cernobbio-intesa-idUSL5N0H306K20130907   \n",
       "9               http://www.reuters.com/article/2013/09/07/weather-hurricanes-idUSL2N0H21LR20130907   \n",
       "\n",
       "                  created_at  \\\n",
       "0  2013-09-07 14:09:11+00:00   \n",
       "1  2013-09-07 13:50:56+00:00   \n",
       "2  2013-09-07 13:11:33+00:00   \n",
       "3  2013-09-07 13:00:00+00:00   \n",
       "4  2013-09-07 11:09:50+00:00   \n",
       "5  2013-09-07 12:34:32+00:00   \n",
       "6  2013-09-07 11:34:40+00:00   \n",
       "7  2013-09-07 12:04:04+00:00   \n",
       "8  2013-09-07 11:49:29+00:00   \n",
       "9  2013-09-07 12:00:00+00:00   \n",
       "\n",
       "                                                           feed_URL  \\\n",
       "0                           http://feeds.reuters.com/news/usmarkets   \n",
       "1                        http://feeds.reuters.com/reuters/bondsNews   \n",
       "2        http://feeds.reuters.com/reuters/cyclicalconsumergoodsNews   \n",
       "3                        http://feeds.reuters.com/reuters/bondsNews   \n",
       "4        http://feeds.reuters.com/reuters/cyclicalconsumergoodsNews   \n",
       "5                           http://feeds.reuters.com/news/usmarkets   \n",
       "6                   http://feeds.reuters.com/reuters/financialsNews   \n",
       "7                      http://feeds.reuters.com/reuters/environment   \n",
       "8  http://feeds.reuters.com/reuters/financialServicesrealEstateNews   \n",
       "9  http://feeds.reuters.com/reuters/financialServicesrealEstateNews   \n",
       "\n",
       "                     insertion_date  \\\n",
       "0  2013-09-07 14:35:16.164000+00:00   \n",
       "1  2013-09-07 14:04:41.071000+00:00   \n",
       "2  2013-09-07 13:22:29.240000+00:00   \n",
       "3  2013-09-07 13:20:11.633000+00:00   \n",
       "4  2013-09-07 12:53:21.464000+00:00   \n",
       "5  2013-09-07 12:53:04.493000+00:00   \n",
       "6  2013-09-07 12:12:56.359000+00:00   \n",
       "7  2013-09-07 12:12:56.359000+00:00   \n",
       "8  2013-09-07 12:12:37.609000+00:00   \n",
       "9  2013-09-07 12:12:37.609000+00:00   \n",
       "\n",
       "                                                                                      tags  \\\n",
       "0                             [{u'term': u'marketsNews', u'scheme': None, u'label': None}]   \n",
       "1                               [{u'term': u'bondsNews', u'scheme': None, u'label': None}]   \n",
       "2             [{u'term': u'cyclicalConsumerGoodsSector', u'scheme': None, u'label': None}]   \n",
       "3                               [{u'term': u'bondsNews', u'scheme': None, u'label': None}]   \n",
       "4             [{u'term': u'cyclicalConsumerGoodsSector', u'scheme': None, u'label': None}]   \n",
       "5                             [{u'term': u'marketsNews', u'scheme': None, u'label': None}]   \n",
       "6                        [{u'term': u'financialsSector', u'scheme': None, u'label': None}]   \n",
       "7                         [{u'term': u'environmentNews', u'scheme': None, u'label': None}]   \n",
       "8  [{u'term': u'rbssFinancialServicesAndRealEstateNews', u'scheme': None, u'label': None}]   \n",
       "9  [{u'term': u'rbssFinancialServicesAndRealEstateNews', u'scheme': None, u'label': None}]   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  text  \\\n",
       "0  [TORONTO, Sept 7  - Forget for a minute the Hugh\\nJackman of Broadway musicals  and \"Les Miserables\" and \"X-Men\"\\nmovies, and envision the Australian actor as a Pennsylvania\\nsurvivalist and desperate father who takes justice into his own\\nhands., As a carpenter without enough work who stockpiles supplies\\nand doesn't trust government, he is the seething vigilante who\\ndrives the dark thriller \"Prisoners,\" one of the most\\ntalked-about films at the Toronto International Film Festival\\nand th...   \n",
       "1                                                                                                                                                                                                                                                                                                                                                  [* Warns of \"disastrous loss of credibility\" if deficit\\ntarget missed, * Berlusconi party pledges \"responsibility\" ahead of key\\nSenate vote, By Giancarlo Navach]   \n",
       "2                                   [By Paola Arosio, MILAN, Sept 7 (Reuters) - Egyptian telecoms tycoon Naguib\\nSawiris said on Saturday he was still interested in taking a\\nstake in Telecom Italia but might be discouraged if\\nthe Italian government was opposed., In an emailed statement to Reuters, Sawiris said he was\\nconsidering a \"potential investment in Telecom Italia\" but might\\nnot submit a proposal \"in view of (Italy's)... alleged\\npreference for Telefonica reported in the Italian press.\"]   \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                             [* City pensions compete with schools for extra cash, * Plans to sell shuttered schools, By Hilary Russ]   \n",
       "4  [The official Xinhua news agency put out a one-line report\\nstating: \"Talk on the Internet about Xi Jinping wearing a\\nwedding ring at the G20 summit is fake information.\" It provided\\nno further explanation., The denial comes as China embarks on yet another crackdown\\non what it terms \"online rumours\", as the control-obsessed\\ngovernment tries once again to rein in social media., Xinhua's report appeared to stem from a lingering shot\\ncarried on state television earlier in the week of Xi's ...   \n",
       "5                                                                                                                                                                                                                                                                                                                                                                                    [* Australian conservatives win power, * Defeated Rudd quits as Labor leader, * Voters punish Labor party for political disunity]   \n",
       "6                                                                                                                                                            [The government has already banned duty-free import of\\nflat-screen televisions from Aug. 26, adding to a package of\\nmeasures designed to prop up the rupee., \"I hope that all these measures taken together will have a\\nbeneficial impact on inflation,\" Chidambaram told lawmakers., He added that no decision has been taken to raise fuel\\nprices.]   \n",
       "7  [MIAMI  - The 2013 Atlantic hurricane season, which forecasters had predicted would be more active than normal, has turned out to be something of a dud so far as an unusual calm hangs over the tropics., As the season heads into the historic peak for activity, it may even enter the record books as marking the quietest start to any Atlantic hurricane season in decades., \"It certainly looks like pretty much of a forecast bust,\" said Jeff Masters, a hurricane expert and director of meteorology a...   \n",
       "8                                                                                                                                                                      [By Lisa Jucca and Paola Arosio, CERNOBBIO, Italy, Sept 7 (Reuters) - The chief executive of\\nIntesa Sanpaolo said on Saturday he expected a new\\nphase of mergers in the banking sector, both inside Italy and\\nacross Europe., Enrico Cucchiani also said his bank was not interested in\\nincreasing its exposure to Italy's banking system.]   \n",
       "9  [MIAMI, Sept 7  - The 2013 Atlantic hurricane\\nseason, which forecasters had predicted would be more active\\nthan normal, has turned out to be something of a dud so far as\\nan unusual calm hangs over the tropics., As the season heads into the historic peak for activity, it\\nmay even enter the record books as marking the quietest start to\\nany Atlantic hurricane season in decades., \"It certainly looks like pretty much of a forecast bust,\"\\nsaid Jeff Masters, a hurricane expert and director of...   \n",
       "\n",
       "                                                                                title  \n",
       "0  'Prisoners' captures rage and despair of a vulnerable America at Toronto film fest  \n",
       "1                            UPDATE 1-Minister says Italy will dodge political crisis  \n",
       "2                      UPDATE 1-Sawiris eyes Telecom Italia, but concerned about Rome  \n",
       "3                     Fiscal crisis looms for Philadelphia schools as students return  \n",
       "4                               Chinese left confused by bizarre missive on Xi's ring  \n",
       "5   RPT-UPDATE 5-Conservative leader Abbott sweeps into power in Australian elections  \n",
       "6                            India to unveil new measures to curb inessential imports  \n",
       "7                                  Atlantic hurricane season - a record-breaking dud?  \n",
       "8                                 INTERVIEW-Intesa CEO expects bank mergers in Europe  \n",
       "9                                  Atlantic hurricane season - a record-breaking dud?  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open( os.path.join(data_directory, 'reuters.json') ) as f:\n",
    "    reuters = json_util.loads(f.read())\n",
    "\n",
    "df_reuters = pd.DataFrame(reuters)\n",
    "df_reuters = df_reuters.drop('_id', axis=1);\n",
    "df_reuters = df_reuters.drop('source', axis=1);\n",
    "df_reuters = df_reuters.sort('insertion_date', ascending=False)\n",
    "df_reuters = df_reuters.reset_index().drop('index', axis=1)\n",
    "\n",
    "df_reuters.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Payrolls rose by 169,000 last month, a Labor Department\\nreport showed yesterday in Washington , and the jobless rate fell\\nto 7.3 percent as people left the workforce. Revisions to prior\\nreports subtracted a total of 74,000 jobs to payrolls in the\\nprevious two months, while hours worked and earnings rose.',\n",
       " u'The jobs report is the last one Fed policy makers will see\\nbefore their Sept. 17-18 meeting, when they resume a debate on\\nwhen to pare $85 billion in monthly bond purchases. The increase\\nin employment was probably strong enough to convince central\\nbankers to reduce so-called quantitative easing by about $10\\nbillion a month, said Stuart Hoffman, chief economist at PNC\\nFinancial Services Group in Pittsburgh.',\n",
       " u'\\u201cMaybe they do this taper lite,\\u201d said Hoffman, the top\\nforecaster of private payroll growth for the past two years,\\naccording to data compiled by Bloomberg. \\u201cThe headcount\\u2019s weak,\\nbut the income earned from wages and longer hours is positive.\\nTo me, it\\u2019s sort of win one, lose one.\\u201d',\n",
       " u'Hoffman\\u2019s forecast is in line with the median estimate in a\\nBloomberg survey of 34 economists after yesterday\\u2019s jobs report\\nshowing the Fed is likely to reduce asset purchases to $75\\nbillion this month. The Federal Open Market Committee will slow\\nTreasury purchases to $35 billion from $45 billion while\\nmaintaining mortgage-bond buying at $40 billion , according to\\nthe survey. That pace was unchanged from an Aug. 9-13 poll.']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bloomberg.text[1]            #.text --> text = column header b/c of dot; indexing to first element"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'TORONTO, Sept 7  - Forget for a minute the Hugh\\nJackman of Broadway musicals  and \"Les Miserables\" and \"X-Men\"\\nmovies, and envision the Australian actor as a Pennsylvania\\nsurvivalist and desperate father who takes justice into his own\\nhands.',\n",
       " u'As a carpenter without enough work who stockpiles supplies\\nand doesn\\'t trust government, he is the seething vigilante who\\ndrives the dark thriller \"Prisoners,\" one of the most\\ntalked-about films at the Toronto International Film Festival\\nand the subject of early Oscar buzz.',\n",
       " u'Playing opposite Jake Gyllenhaal\\'s small town detective,\\nJackman\\'s Keller Dover embodies what French Canadian director\\nDenis Villeneuve calls a \"lack of confidence in the\\ninstitutions.\"']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reuters.text[0]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "goals:\n",
    "1. clean text\n",
    "    a. steal useful regular expressions: http://ravikiranj.net/drupal/201205/code/machine-learning/how-build-twitter-sentiment-analyzer\n",
    "2. run NLTK:\n",
    "    a. find named entities\n",
    "    b. find company names\n",
    "    c. find topics\n",
    "    d. synthesize adjectives\n",
    "    e. cluster articles from both Reuters and Bloomberg, group similar articles\n",
    "    f. get POS tags - what can I do with this?\n",
    "3. AFTER 2 is complete\n",
    "    a. look at bigrams for each text block\n",
    "    b. take TF-IDF across documents\n",
    "    c. find unique bigrams/trigrams\n",
    "    d. count unique bigrams/trigrams across documents in discrete time-buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tag.stanford import StanfordTagger\n",
    "from nltk.collocations import *\n",
    "from text.blob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "articles = df_bloomberg.text.tolist()     #.tolist() --> changes type from str to list\n",
    "text = '  '.join(sentence for sentence in articles[0])  #join sentences into 1 string sep by dbl space\n",
    "#text = text.encode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_entities(chunks):\n",
    "    \"given list of tagged parts of speech, returns unique named entities\"\n",
    "\n",
    "    def traverse(tree):\n",
    "        \"recursively traverses an nltk.tree.Tree to find named entities\"\n",
    "        entity_names = []\n",
    "    \n",
    "        if hasattr(tree, 'node') and tree.node:\n",
    "            print tree.node\n",
    "            if tree.node == 'GPE':\n",
    "                entity_names.append(' '.join([child[0] for child in tree]))\n",
    "            elif tree.node == 'PERSON':\n",
    "                entity_names.append(' '.join([child[0] for child in tree]))\n",
    "            else:\n",
    "                for child in tree:\n",
    "                    entity_names.extend(traverse(child))\n",
    "    \n",
    "        return entity_names\n",
    "    \n",
    "    named_entities = []\n",
    "    \n",
    "    for chunk in chunks:\n",
    "        entities = sorted(list(set([word for tree in chunk\n",
    "                            for word in traverse(tree)])))\n",
    "        for e in entities:\n",
    "            if e not in named_entities:\n",
    "                named_entities.append(e)\n",
    "    return named_entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPE\n",
      "GPE\n",
      "PERSON\n",
      "PERSON\n",
      "PERSON\n",
      "PERSON\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[u'Brooklyn', u'New York', u'Merrill', u'Cara E. Greene', u'Merrill Lynch']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " ## takes text and associate w. POS tags (so can filter on basis of tags)\n",
    "\n",
    "tokens = nltk.word_tokenize(text)            \n",
    "sentences = nltk.sent_tokenize(text)\n",
    "words     = (nltk.word_tokenize(sentence) for sentence in sentences)\n",
    "tags       = [nltk.pos_tag(word) for word in words]\n",
    "\n",
    "named_entity_chunks = nltk.batch_ne_chunk(tags)\n",
    "find_entities(named_entity_chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# http://stackoverflow.com/questions/12118720/python-tf-idf-cosine-to-find-document-similarity/12128777\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.metrics.pairwise import linear_kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Lawyers representing female financial advisers and licensed\\ntrainees filed a request for approval of the settlement\\nyesterday in federal court in Brooklyn , New York .',\n",
       " u'The advisers\\nalleged in the lawsuit that Merrill discriminated against women\\nin those positions in compensation and business opportunities.',\n",
       " u'\\u201cThis settlement helps ensure that Merrill Lynch is a\\nplace where women can thrive and be successful,\\u201d Cara E.\\nGreene, a lawyer for the plaintiffs, said in a statement.',\n",
       " u'\\u201cHopefully others will follow Merrill Lynch\\u2019s example.\\u201d  The accord, which is subject to approval by a federal\\njudge, covers about 4,800 women who worked at the firm during\\nthe period of Aug. 2, 2007 to Sept. 15, 2013, according to\\ndocuments filed with the plaintiffs\\u2019 motion.',\n",
       " u'Along with a cash fund, the settlement would require\\nchanges at the firm to be overseen by an independent monitor,\\naccording to the plaintiffs\\u2019 attorneys.']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'cleaned_text' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-561523c339c6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtfidf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTfidfVectorizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mngram_range\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_pattern\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mr'\\b\\w+\\b'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_df\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcleaned_text\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# need to use character n-grams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'cleaned_text' is not defined"
     ]
    }
   ],
   "source": [
    "tfidf = TfidfVectorizer(ngram_range=(1,3), token_pattern=r'\\b\\w+\\b', min_df=1).fit_transform(cleaned_text) # need to use character n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,    8, 3442, 2201,   63,  320, 1371, 1363, 3218, 4599, 3386,\n",
       "       6512, 6515, 4816,  603, 5771,    9, 2687, 2142])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarities = linear_kernel(tfidf[0:1], tfidf).flatten()\n",
    "\n",
    "related_docs_indices = cosine_similarities.argsort()[:-20:-1]\n",
    "related_docs_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.        ,  0.43069066,  0.28236834,  0.17565869,  0.17336819,\n",
       "        0.152429  ,  0.15153243,  0.15124149,  0.15110318,  0.14708165,\n",
       "        0.14708165,  0.14627424,  0.14622723,  0.14495305,  0.13674133,\n",
       "        0.1356015 ,  0.12915598,  0.1270037 ,  0.12179412])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarities[related_docs_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AT_USER $aapl. apple's iphone has cracked.\n",
      "apple's iphone has cracked the last two major airtime provider holdouts ~ URL $aapl\n",
      "report: apple’s iphone has over 43% of the u.s. market $aapl URL\n",
      "$aapl docomo to offer apple's iphone: nikkei URL\n",
      "features of latest apple’s iphone leaked URL $aapl\n",
      "apple's iphone holds 40% share of us smartphone market $aapl URL\n",
      "apple's iphone 5c, a detonator in global smartphone market $aapl URL\n",
      "AT_USER apple's iphone 5c, a detonator in global smartphone market $aapl URL\n",
      "a shareholders guide to apple's iphone trade-in program $aapl URL\n",
      "AT_USER $aapl\n",
      "AT_USER $aapl\n",
      "AT_USER apple's iphone grows to 43% share among us smartphones $aapl URL\n",
      "apple's iphone grows to 43% share among us smartphones $aapl URL\n",
      "AT_USER $aapl apple sep 11 event in beijing\n",
      "$aapl apple’s low-cost iphones shipping to china mobile URL\n",
      "latest parts leak may show apple's 'iphone 5s' fingerprint scanner $aapl URL\n",
      "$aapl apple's worst kept secret: bigger iphones in 2014 URL\n",
      "$aapl new high-res pics may again show apple's 'iphone 5s' fingerprint sensor URL\n",
      "$aapl japan's largest wirless carrier, docomo, to carry apple's iphone starting this fall - report URL\n"
     ]
    }
   ],
   "source": [
    "for i in related_docs_indices:\n",
    "    print cleaned_text[i]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Using cosine similarity as a distance function: we simply don't have enough characters to properly cluster tweets"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.3160494 ,  0.2486245 ,  0.12166814, ...,  0.2115839 ,\n",
       "        0.20560519,  0.05370148])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(min_df=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x6 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 6 stored elements in Compressed Sparse Column format>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.fit_transform([text[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5, stop_words='english')\n",
    "#X_train = vectorizer.fit_transform(data_train.data)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Tag\n",
    "Description\n",
    "CC\tCoordinating conjunction\n",
    "CD\tCardinal number\n",
    "DT\tDeterminer\n",
    "EX\tExistential there\n",
    "FW\tForeign word\n",
    "IN\tPreposition or subordinating conjunction\n",
    "JJ\tAdjective\n",
    "JJR\tAdjective, comparative\n",
    "JJS\tAdjective, superlative\n",
    "LS\tList item marker\n",
    "MD\tModal\n",
    "NN\tNoun, singular or mass\n",
    "NNS\tNoun, plural\n",
    "NNP\tProper noun, singular\n",
    "NNPS\tProper noun, plural\n",
    "PDT\tPredeterminer\n",
    "POS\tPossessive ending\n",
    "PRP\tPersonal pronoun\n",
    "PRP$\tPossessive pronoun\n",
    "RB\tAdverb\n",
    "RBR\tAdverb, comparative\n",
    "RBS\tAdverb, superlative\n",
    "RP\tParticle\n",
    "SYM\tSymbol\n",
    "TO\tto\n",
    "UH\tInterjection\n",
    "VB\tVerb, base form\n",
    "VBD\tVerb, past tense\n",
    "VBG\tVerb, gerund or present participle\n",
    "VBN\tVerb, past participle\n",
    "VBP\tVerb, non-3rd person singular present\n",
    "VBZ\tVerb, 3rd person singular present\n",
    "WDT\tWh-determiner\n",
    "WP\tWh-pronoun\n",
    "WP$\tPossessive wh-pronoun\n",
    "WRB\tWh-adverb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.stem import PorterStemmer as porter_stemmer\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lemmatize_text(text):\n",
    "\ttokens = nltk.word_tokenize(text)\n",
    "\tsentences = nltk.sent_tokenize(text)\n",
    "\twords = (nltk.word_tokenize(sentence) for sentence in sentences)\n",
    "\ttag_tuples = [nltk.pos_tag(word) for word in words]\n",
    "\n",
    "\tnew_sentence = []\n",
    "\tfor tuples in tag_tuples:\n",
    "\t\tfor word, tag in tuples:\n",
    "\t\t\tif tag.startswith('V'):\n",
    "\t\t\t\tnew_sentence.append( lemmatizer.lemmatize(word, wn.VERB) )\n",
    "\t\t\t\t\n",
    "\t\t\telif tag.startswith('J'):\n",
    "\t\t\t\tnew_sentence.append( lemmatizer.lemmatize(word, wn.ADJ) )\n",
    "\t\t\t\t\n",
    "\t\t\telif tag.startswith('N'):\n",
    "\t\t\t\tnew_sentence.append( lemmatizer.lemmatize(word, wn.NOUN) )\n",
    "\t\t\t\t\n",
    "\t\t\telif tag.startswith('R'):\n",
    "\t\t\t\tnew_sentence.append( lemmatizer.lemmatize(word, wn.ADV) )\n",
    "\t\t\t\t\n",
    "\t\t\telse:\n",
    "\t\t\t\tnew_sentence.append( word )\n",
    "\n",
    "\tprint text\n",
    "\tprint ' '.join(i for i in new_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "will low cost iphones actually lower apple margins? $aapl URL\n",
      "will low cost iphones actually low apple margin ? $ aapl URL\n"
     ]
    }
   ],
   "source": [
    "lemmatize_text(cleaned_text[5])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Fun with the CMU tagger\n",
    "\n",
    "https://github.com/brendano/ark-tweet-nlp\n",
    "http://www.ark.cs.cmu.edu/TweetNLP/\n",
    "https://github.com/ianozsvald/ark-tweet-nlp-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import CMUTweetTagger # will wrap this with a web-service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AT_USER $aapl. apple's iphone has cracked.\n",
      "[[('AT_USER', 'P', 0.5752), ('$aapl', '^', 0.7174), ('.', ',', 0.9668), (\"apple's\", 'Z', 0.6764), ('iphone', '^', 0.7309), ('has', 'V', 0.9833), ('cracked', 'V', 0.5413), ('.', ',', 0.9983)]]\n",
      "\n",
      "$aapl is holding well in the bull flag. did you notice the golden cross on the daily? ;) URL\n",
      "[[('$aapl', '^', 0.8645), ('is', 'V', 0.9961), ('holding', 'V', 0.9728), ('well', 'R', 0.8528), ('in', 'P', 0.9986), ('the', 'D', 0.9991), ('bull', 'N', 0.9745), ('flag', 'N', 0.9849), ('.', ',', 0.9979), ('did', 'V', 0.9994), ('you', 'O', 0.9957), ('notice', 'V', 0.9922), ('the', 'D', 0.999), ('golden', 'A', 0.4243), ('cross', 'N', 0.9899), ('on', 'P', 0.9987), ('the', 'D', 0.9991), ('daily', 'A', 0.5749), ('?', ',', 0.9897), (';)', 'E', 0.9774), ('URL', 'N', 0.4083)]]\n",
      "\n",
      "$aapl if we get a pullback, load the house, don't listen to cnbc, they'll tell you to buy once it hits 560 and they exit\n",
      "[[('$aapl', '^', 0.7156), ('if', 'P', 0.9991), ('we', 'O', 0.9988), ('get', 'V', 0.9999), ('a', 'D', 0.9979), ('pullback', 'N', 0.9992), (',', ',', 0.9969), ('load', 'V', 0.6802), ('the', 'D', 0.9991), ('house', 'N', 0.9824), (',', ',', 0.9988), (\"don't\", 'V', 0.9995), ('listen', 'V', 0.9999), ('to', 'P', 0.982), ('cnbc', '^', 0.8518), (',', ',', 0.9987), (\"they'll\", 'L', 0.9564), ('tell', 'V', 0.9997), ('you', 'O', 0.9995), ('to', 'P', 0.9889), ('buy', 'V', 0.9972), ('once', 'R', 0.911), ('it', 'O', 0.9909), ('hits', 'V', 0.9862), ('560', '$', 0.9698), ('and', '&', 0.9985), ('they', 'O', 0.9624), ('exit', 'V', 0.8208)]]\n",
      "\n",
      "AT_USER lets come back to the socks and $aapl in a week and i will buy you a ticket to stocktoberfest if apple is lower next friday\n",
      "[[('AT_USER', 'P', 0.3646), ('lets', 'L', 0.8862), ('come', 'V', 0.9995), ('back', 'R', 0.9183), ('to', 'P', 0.9992), ('the', 'D', 0.9995), ('socks', 'N', 0.888), ('and', '&', 0.9962), ('$aapl', '^', 0.4977), ('in', 'P', 0.9824), ('a', 'D', 0.9976), ('week', 'N', 0.9927), ('and', '&', 0.9986), ('i', 'O', 0.9927), ('will', 'V', 0.9996), ('buy', 'V', 0.9996), ('you', 'O', 0.9963), ('a', 'D', 0.9971), ('ticket', 'N', 0.998), ('to', 'P', 0.9974), ('stocktoberfest', 'V', 0.872), ('if', 'P', 0.9951), ('apple', '^', 0.7237), ('is', 'V', 0.9943), ('lower', 'A', 0.8732), ('next', 'A', 0.5088), ('friday', '^', 0.5306)]]\n",
      "\n",
      "$aapl golden cross coming next week $525 by tuesday and 5⃣5⃣0⃣ by end of next week.\n",
      "[[('$aapl', '^', 0.6884), ('golden', '^', 0.6076), ('cross', 'N', 0.6686), ('coming', 'V', 0.9929), ('next', 'A', 0.8788), ('week', 'N', 0.993), ('$525', '$', 0.9744), ('by', 'P', 0.9955), ('tuesday', '^', 0.9903), ('and', '&', 0.9986), ('5\\xe2\\x83\\xa35\\xe2\\x83\\xa30\\xe2\\x83\\xa3', '$', 0.6779), ('by', 'P', 0.9937), ('end', 'N', 0.9651), ('of', 'P', 0.9931), ('next', 'A', 0.9685), ('week', 'N', 0.9985), ('.', ',', 0.9981)]]\n",
      "\n",
      "will low cost iphones actually lower apple margins? $aapl URL\n",
      "[[('will', 'V', 0.9824), ('low', 'A', 0.9878), ('cost', 'N', 0.9826), ('iphones', 'N', 0.7908), ('actually', 'R', 0.9996), ('lower', 'A', 0.97), ('apple', 'N', 0.8366), ('margins', 'N', 0.9974), ('?', ',', 0.9975), ('$aapl', '^', 0.3478), ('URL', '^', 0.6231)]]\n",
      "\n",
      "thanks for july and august $aapl . i'm headed to maui. next week better be good! sell order at $520...\n",
      "[[('thanks', 'N', 0.8631), ('for', 'P', 0.9998), ('july', '^', 0.985), ('and', '&', 0.998), ('august', '^', 0.6996), ('$aapl', '^', 0.917), ('.', ',', 0.9941), (\"i'm\", 'L', 0.9968), ('headed', 'V', 0.9775), ('to', 'P', 0.9969), ('maui', '^', 0.923), ('.', ',', 0.9961), ('next', 'A', 0.9548), ('week', 'N', 0.9888), ('better', 'R', 0.8012), ('be', 'V', 0.9641), ('good', 'A', 0.9926), ('!', ',', 0.9975), ('sell', 'V', 0.9075), ('order', 'N', 0.8638), ('at', 'P', 0.9987), ('$520', '$', 0.86), ('...', ',', 0.5878)]]\n",
      "\n",
      "$aapl u can make good money by doing the opposite of what fm crew is saying. maybe that's why some ppl r still watching them.\n",
      "[[('$aapl', '^', 0.4542), ('u', 'O', 0.9962), ('can', 'V', 0.9996), ('make', 'V', 0.9983), ('good', 'A', 0.9988), ('money', 'N', 0.9982), ('by', 'P', 0.9986), ('doing', 'V', 0.9995), ('the', 'D', 0.9997), ('opposite', 'N', 0.9028), ('of', 'P', 0.9979), ('what', 'O', 0.867), ('fm', 'N', 0.8343), ('crew', 'N', 0.9994), ('is', 'V', 0.9977), ('saying', 'V', 0.9507), ('.', ',', 0.9988), ('maybe', 'R', 0.9702), (\"that's\", 'L', 0.9902), ('why', 'R', 0.9775), ('some', 'D', 0.972), ('ppl', 'N', 0.9842), ('r', 'V', 0.9123), ('still', 'R', 0.9958), ('watching', 'V', 0.9999), ('them', 'O', 0.9852), ('.', ',', 0.9957)]]\n",
      "\n",
      "apple's iphone has cracked the last two major airtime provider holdouts ~ URL $aapl\n",
      "[[(\"apple's\", 'Z', 0.3352), ('iphone', '^', 0.7342), ('has', 'V', 0.9849), ('cracked', 'V', 0.9677), ('the', 'D', 0.9997), ('last', 'A', 0.9929), ('two', '$', 0.9535), ('major', 'A', 0.9888), ('airtime', 'N', 0.9974), ('provider', 'N', 0.9717), ('holdouts', 'N', 0.9971), ('~', 'G', 0.6595), ('URL', '^', 0.5763), ('$aapl', '^', 0.8882)]]\n",
      "\n",
      "$aapl apple's worst kept secret: bigger iphones in 2014 URL\n",
      "[[('$aapl', '^', 0.8455), (\"apple's\", 'Z', 0.8228), ('worst', 'A', 0.97), ('kept', 'V', 0.8924), ('secret', 'N', 0.5294), (':', ',', 0.9501), ('bigger', 'A', 0.997), ('iphones', 'N', 0.9168), ('in', 'P', 0.9973), ('2014', '$', 0.9575), ('URL', 'N', 0.9012)]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for text in cleaned_text[:10]:\n",
    "    print text\n",
    "    print CMUTweetTagger.runtagger_parse([text])\n",
    "    print"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "interesting projects:\n",
    "https://github.com/ianozsvald/social_media_brand_disambiguator\n",
    "https://github.com/ianozsvald/twitter-text-python\n",
    "https://github.com/ianozsvald/twitter_networkx_concept_map\n",
    "\n",
    "http://blog.yhathq.com/posts/named-entities-in-law-and-order-using-nlp.html\n",
    "http://blog.newsle.com/2013/02/01/text-classification-and-feature-hashing-sparse-matrix-vector-multiplication-in-cython/\n",
    "https://textblob.readthedocs.org/en/latest/\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}