From e7191f1a29ab52bcfe4145f98ec886c69229b8b2 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Thu, 21 Dec 2017 11:20:21 +0200 Subject: [PATCH] scraderdata --- algorithm.py | 22 ++--- scraderdata.csv | 243 ++++++++++++++++++++++++++++++++++++++++++++++++ script.py | 2 +- 3 files changed, 255 insertions(+), 12 deletions(-) create mode 100644 scraderdata.csv diff --git a/algorithm.py b/algorithm.py index b959fb2..c8afa52 100644 --- a/algorithm.py +++ b/algorithm.py @@ -68,18 +68,18 @@ def train_predict(clf, X_train, y_train, X_test, y_test): def run_algorithm(filename): - data = pd.read_csv('./comb8.csv', sep=',', encoding='utf-8') - data1 = data['Article'] - data2 = data['Sentiment'] + data = pd.read_csv('./scraderdata.csv', sep=',', encoding='utf-8') + data1 = data['title'] + data2 = data['direction'] print 'Data read Successfully' data = pd.concat([data1.reset_index(drop=True), data2], axis=1) - data.columns = ['Articles', 'Sentiment'] - data.groupby(['Sentiment']).count().reset_index() - ntrain = len(data['Articles'])*0.8 + data.columns = ['title', 'direction'] + data.groupby(['direction']).count().reset_index() + ntrain = len(data['title'])*0.8 ntrain = int(round(ntrain)) train_set = data[:ntrain] - train_set.groupby(['Sentiment']).count().reset_index() + train_set.groupby(['direction']).count().reset_index() data = data.sample(frac=1) g = TfidfVectorizer(min_df=5, max_df=1000, ngram_range=(1, 6), stop_words=[ @@ -97,10 +97,10 @@ def run_algorithm(filename): "Qualcomm", "Honeywell", "ABB", "Astrazeneca", "Carrefour", "Canon", "Canon", "Aetna" ], analyzer=u'word', max_features=5000) - X_train = g.fit_transform(data['Articles'][:ntrain]).toarray() - X_test = g.transform(data['Articles'][ntrain:]).toarray() - y_train = data['Sentiment'][:ntrain] - y_test = data['Sentiment'][ntrain:] + X_train = g.fit_transform(data['title'][:ntrain]).toarray() + X_test = g.transform(data['title'][ntrain:]).toarray() + y_train = data['direction'][:ntrain] + y_test = data['direction'][ntrain:] clf = SVC(kernel='linear', probability=True) train_predict(clf, X_train, y_train, X_test, y_test) diff --git a/scraderdata.csv b/scraderdata.csv new file mode 100644 index 0000000..44c0abf --- /dev/null +++ b/scraderdata.csv @@ -0,0 +1,243 @@ +,User,appended,checked,company,direction,image_url,item_url,subtitle,title,website,website_url +,,TRUE,TRUE,,POS,,,,HP's stock set a new 52-week high on Friday after delivering a strong profit forecast,, +,,TRUE,TRUE,,POS,,,,This Google engineer teaches his coworkers how to fly airplanes ... and it's changed his career in surprising ways,, +,,TRUE,TRUE,,POS,,,,"Dropbox just signed the biggest office space deal in San Francisco history — 736,000 square feet",, +,,TRUE,TRUE,,POS,,,,"The CEO of GitHub, which caters to coders, thinks automation will bring an end to traditional software programming",, +,,TRUE,TRUE,,POS,,,,"Bill.com is the latest fintech heavyweight, with a $742.8 million valuation and big-name backers",, +,,TRUE,TRUE,,POS,,,,Microsoft is holding a $3.5 million competition to find the next great garage startup — 'like American Idol',, +,,TRUE,TRUE,,NEG,,,,Microsoft's dream of Windows on phones is officially dead — now it's time for Plan B,, +,,TRUE,TRUE,,POS,,,,A top Microsoft exec says the idea it will kill its Surface gadget business is 'so far from the truth',, +,,TRUE,TRUE,,POS,,,,IBM wants lawmakers to protect dreamers by the end of the year and is going all out to persuade them,, +,,TRUE,TRUE,,NEG,,,,$1.6 billion startup MongoDB could be worth less than $1 billion when its IPO happens next week,, +,,TRUE,TRUE,,POS,,,,An underground Silicon Valley sensation just got $20 million from Google's venture arm to take over the world,, +,,TRUE,TRUE,,NEG,,,,Amazon Top Exec Roy Price on Leave After Sexual Harassment Allegations,, +,,TRUE,TRUE,,NEG,,,,Couple Finds Hidden Camera in Their Airbnb,, +,,TRUE,TRUE,,NEG,,,,'I’m Not Just Some Silent Victim.' The Model Featured in Dove's Controversial Ad Speaks Out,, +,,TRUE,TRUE,,NEG,,,,Dove's 'Racist' Ad Isn't the First Time the Company Was Criticized for Being Offensive,, +,,TRUE,TRUE,,POS,,,,IRS Gives Equifax Multimillion-Dollar Deal Despite Massive Data Breach,, +,,TRUE,TRUE,,NEG,,,,'You Should Be Fired.' Elizabeth Warren Slams Wells Fargo CEO Over Fake Accounts,, +,,TRUE,TRUE,,NEG,,,,Former Equifax CEO Gets Ripped By Congress Over Massive Data Breach,, +,,TRUE,TRUE,,NEG,,,,Equifax Says 2.5 Million More People May Have Been Affected by Security Breach,, +,,TRUE,TRUE,,NEG,,,,Equifax CEO Richard Smith Ousted Following Massive Data Breach,, +,,TRUE,TRUE,,NEG,,,,'We're Getting Our Roads Back.' London's Black Cab Drivers Hail Uber Ban,, +,,TRUE,TRUE,,NEG,,,,Equifax Got Hacked Nearly Five Months Before It Previously Said,, +,,TRUE,TRUE,,NEG,,,,Top Equifax Executives Announce Immediate Retirement After Massive Data Breach,, +,,TRUE,TRUE,,NEG,,,,Equifax Could Have Fixed the Software Flaw That Led to Massive Data Theft,, +,,TRUE,TRUE,,POS,,,,Nestlé Just Announced a Huge Blue Bottle Coffee Deal. What's That?,, +,,TRUE,TRUE,,NEG,,,,Vanity Fair Editor and Donald Trump Antagonist Graydon Carter Stepping Down,, +,,TRUE,TRUE,,POS,,,,Edward Felsenthal Chosen To Lead TIME as New Editor,, +,,TRUE,TRUE,,POS,,,,Here's How Much Uber's New CEO Dara Khosrowshahi Is Worth,, +,,TRUE,TRUE,,NEG,,,,Samsung Heir Lee Jae-yong Jailed for Five Years for Corruption,, +,,TRUE,TRUE,,POS,,,,Google Cancels Web Domain Registration Neo-Nazi Site The Daily Stormer,, +,,TRUE,TRUE,,POS,,,,The Next Apple Watch Will Have a Feature Runners Will Love,, +,,TRUE,TRUE,,NEG,,,,2 Lawsuits Against KFC Accuse Manager of Sexually Assaulting Teen Employees,, +,,TRUE,TRUE,,POS,,,,How Toymaker Mattel Plans to Win Over iPad-Obsessed Kids,, +,,TRUE,TRUE,,POS,,,,Pringles Makes Instant Noodles Even Easier to Eat With Ramen-Flavored Chip,, +,,TRUE,TRUE,,POS,,,,How Apple's iPhone Changed These 5 Major Industries,, +,,TRUE,TRUE,,NEG,,,,Uber's Next CEO Has to Face This Huge Challenge,, +,,TRUE,TRUE,,NEG,,,,Uber Co-Founder Travis Kalanick Has Resigned as CEO,, +,,TRUE,TRUE,,POS,,,,Google Wants to Help You Search for a New Job,, +,,TRUE,TRUE,,POS,,,,Nutella Maker Wins Case Against Rival Chocolate Spread That Claims It's Much Healthier,, +,,TRUE,TRUE,,POS,,,,How Whole Foods Got So Big,, +,,TRUE,TRUE,,NEG,,,,Uber Director Resigns After Making an 'Inappropriate' Comment About Women,, +,,TRUE,TRUE,,NEG,,,,Uber Board Member Apologizes for 'Disrespectful' Joke About Women at Sexual Harassment Meeting,, +,,TRUE,TRUE,,POS,,,,Apple Just Confirmed Rumors It's Working On Something Huge,, +,,TRUE,TRUE,,POS,,,,This Is Apple's Insanely Powerful New iMac Pro,, +,,TRUE,TRUE,,POS,,,,Watch Live: Apple Makes Big Announcements at WWDC 2017,, +,,TRUE,TRUE,,POS,,,,Everything Sony Told Us About the Future of PlayStation,, +,,TRUE,TRUE,,POS,,,,What LinkedIn's Co-Founder Thinks About Workplace Automation,, +,,TRUE,TRUE,,POS,,,,"Sony's PlayStation 3 Lives On, Despite Retirement",, +,,TRUE,TRUE,,POS,,,,WayUp Is a Booming Job-Hunting Site for Millennials,, +,,TRUE,TRUE,,NEG,,,,U.S. Sues Fiat Chrysler for Using Software to Bypass Diesel Emission Controls,, +,,TRUE,TRUE,,POS,,,,Nintendo Switch and 'Mario Kart 8 Deluxe' Dominated April Game Sales,, +,,TRUE,TRUE,,NEG,,,,Snapchat's Stock Plummets Nearly 25% Following Its First Earnings Report,, +,,TRUE,TRUE,,NEG,,,,Ben & Jerry's Recalls Pint Slices Due to Possible Peanut Contamination,, +,,TRUE,TRUE,,NEG,,,,Watch Live: United Airlines CEO Oscar Munoz Testifies Before Congress,, +,,TRUE,TRUE,,POS,,,,Here's What Nintendo Told Us About the New 2DS XL,, +,,TRUE,TRUE,,POS,,,,Nintendo Says it Sold Over 2 Million NES Classics,, +,,TRUE,TRUE,,NEG,,,,United Passenger David Dao Settles With Airline Over Dragging Incident,, +,,TRUE,TRUE,,POS,,,,"Heineken Ad Pairs Up Strangers With Opposing Views on Transgender Rights, Climate Change and Feminism",, +,,TRUE,TRUE,,NEG,,,,Man Gets Kicked off Delta Flight for Using the Restroom,, +,,TRUE,TRUE,,POS,,,,The Switch Is So Successful Even Nintendo Is Surprised,, +,,TRUE,TRUE,,POS,,,,Instagram Just Keeps Growing Like Crazy,, +,,TRUE,TRUE,,NEG,,,,Chobani CEO Hamdi Ulukaya on Lawsuit Against Alex Jones: No Option But to Let the Courts Decide,, +,,TRUE,TRUE,,NEG,,,,The Treasury Department Won't Give Exxon Mobil a Waiver to Drill in Russia,, +,,TRUE,TRUE,,NEG,,,,The FTC Doesn't Want You to Fall for This Nintendo Switch Scam,, +,,TRUE,TRUE,,POS,,,,The Switch Is Nintendo's 'Fastest Selling' System in History,, +,,TRUE,TRUE,,NEG,,,,United Airlines Will Reimburse All Passengers on Flight Where Man Was Forcibly Removed,, +,,TRUE,TRUE,,NEG,,,,Communications Chief Rachel Whetstone Is the Latest Uber Executive to Jump Ship,, +,,TRUE,TRUE,,NEG,,,,United Airlines CEO: Passenger Removed From Flight Was 'Disruptive and Belligerent',, +,,TRUE,TRUE,,NEG,,,,Uber Drivers Aren't Worried About Self-Driving Cars — Yet,, +,,TRUE,TRUE,,NEG,,,,Twitter Drops Lawsuit Against U.S. Government Over Anti-Trump Account,, +,,TRUE,TRUE,,POS,,,,Jeff Bezos Is Selling Amazon Stock So People Can Travel to Space Sooner,, +,,TRUE,TRUE,,POS,,,,Chipotle Is Using a Rapping Cartoon Tortilla to Promote Its Preservative-Free Food,, +,,TRUE,TRUE,,POS,,,,Ford Is Investing $1.2 Billion in U.S. Factories. Find Out Where Your Car Was Made,, +,,TRUE,TRUE,,POS,,,,Ford Announces $1.2 Billion Investment in Michigan Plants,, +,,TRUE,TRUE,,NEG,,,,United Airlines to Customers: ‘Your Leggings Are Welcome’,, +,,TRUE,TRUE,,NEG,,,,United Wouldn't Let 2 Girls on a Plane Because It Apparently Has a Leggings Ban,, +,,TRUE,TRUE,,NEG,,,,Uber's Leadership Says the Company 'Must Change' After Scandals,, +,,TRUE,TRUE,,POS,,,,Nintendo Plans to Double Switch Production after Strong Sales,, +,,TRUE,TRUE,,NEG,,,,Here's What Uber Drivers Actually Think About the Company's PR Nightmares,, +,,TRUE,TRUE,,NEG,,,,A McDonald’s Tweet Called Donald Trump 'a Disgusting Excuse of a President',, +,,TRUE,TRUE,,POS,,,,Here's How You Can Spend a Boozy Night in the Guinness Storehouse in Dublin,, +,,TRUE,TRUE,,NEG,,,,The Latest Victim of Uber's Disruption May Be Itself,, +,,TRUE,TRUE,,NEG,,,,"The Volkswagen Emissions Scandal Could Shorten Thousands of Lives, Study Says",, +,,TRUE,TRUE,,POS,,,,Nintendo Switches Up Mobile Gaming With a Novel Console,, +,,TRUE,TRUE,,POS,,,,Craigslist Founder Announces $1-Million Grant For ProPublica,, +,,TRUE,TRUE,,NEG,,,,Uber Is Facing a Leadership Crisis That Could Cause Lasting Damage,, +,,TRUE,TRUE,,NEG,,,,Uber’s CEO Apologizes After Fight With Driver Is Caught on Video: 'I Must...Grow Up',, +,,TRUE,TRUE,,NEG,,,,Uber Hires Former Attorney-General Eric Holder to Review Sexual-Harassment Claims,, +,,TRUE,TRUE,,NEG,,,,Tim Cook Just Bought Himself More Time to Prove Apple Doubters Wrong,, +,,TRUE,TRUE,,POS,,,,Ford CEO Criticizes President Trump's Immigration Ban: 'We Do Not Support This Policy',, +,,TRUE,TRUE,,POS,,,,Ryan Lochte Seeks Redemption As New PowerBar Spokesman,, +,,TRUE,TRUE,,POS,,,,Facebook Launches 'Journalism Project' to Improve Media Ties,, +,,TRUE,TRUE,,POS,,,,Under Armour Wants You to Buy Tom Brady's Pajamas,, +,,TRUE,TRUE,,NEG,,,,Credit Suisse to Pay Out $5 Billion Over Toxic Mortgages,, +,,TRUE,TRUE,,NEG,,,,Walmart to Stop Selling ‘Bulletproof: Black Lives Matter’ T-Shirts,, +,,TRUE,TRUE,,NEG,,,,Why the Latest Yahoo Hack Is So Much Worse Than You Think,, +,,TRUE,TRUE,,POS,,,,You Can Finally Order Apple's Wireless AirPod Earbuds,, +,,TRUE,TRUE,,POS,,,,Facebook Is Looking to Hire a 20-Year Media Veteran to Help it Rethink News,, +,,TRUE,TRUE,,POS,,,,Sony's PlayStation 4 Just Passed 50 Million Units Sold,, +,,TRUE,TRUE,,NEG,,,,Roy Price Quits Amazon Studios After Sexual Harassment Claim,, +,,TRUE,TRUE,,NEG,,,,Google Serves Fake News Ads in an Unlikely Place: Fact-Checking Sites,, +,,TRUE,TRUE,,NEG,,,,Google Maps Pulls Calorie-Counting Feature After Criticism,, +,,TRUE,TRUE,,POS,,,,Google’s Pixel 2 Smartphone Is a Powerful Extravagance ,, +,,TRUE,TRUE,,NEG,,,,Samsung Is Under Scrutiny Again as South Korean Police Raid Offices,, +,,TRUE,TRUE,,POS,,,,Netflix Says It Will Spend Up to $8 Billion on Content Next Year,, +,,TRUE,TRUE,,NEG,,,,BP eyes smaller renewable investments to avoid repeating losses,, +,,TRUE,TRUE,,NEG,,,,EU air safety body urges halt on use of Kobe Steel products,, +,,TRUE,TRUE,,POS,,,,JPMorgan to buy online payment provider WePay,, +,,TRUE,TRUE,,POS,,,,IBM beats revenue estimates; hints at sales growth,, +,,TRUE,TRUE,,NEG,,,,Malaysia Airlines just lost another CEO,, +,,TRUE,TRUE,,POS,,,,Uber will still operate in Quebec -- for now ,, +,,TRUE,TRUE,,NEG,,,,Apple's case against the FBI won't be easy ,, +,,TRUE,TRUE,,POS,,,,Apple's next goal: an unbreakable iPhone,, +,,TRUE,TRUE,,NEG,,,,Unicorn once worth $2.7 billion collapses into bankruptcy ,, +,,TRUE,TRUE,,NEG,,,,Apple's stock has worms but FBI isn't one of them ,, +,,TRUE,TRUE,,POS,,,,Walmart yanks hoverboards from its site,, +,,TRUE,TRUE,,POS,,,,David Pogue tests 4K streaming boxes,, +,,TRUE,TRUE,,NEG,,,,"BMW Recalls 1 Million Vehicles Over Fire Risk, Recommends Parking Outside - NBC News",, +,,TRUE,TRUE,,POS,,,,Report: Apple revamped overseas ops to find new tax havens - ABC News,, +,,TRUE,TRUE,,POS,,,,Google Pixel 2 review,, +,,TRUE,TRUE,,POS,,,,PlayerUnknown's Battlegrounds using Microsoft Azure and Amazon Web Services - Business Insider,, +,,TRUE,TRUE,,POS,,,,Microsoft Word gets Resume Assistant with LinkedIn to help you write a killer resume - Business Insider,, +,,TRUE,TRUE,,POS,,,,Uber says it will bring its flying taxis to Los Angeles in 2020 - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,Uber tops this surprising list of the 50 most desirable startups - Business Insider,, +,,TRUE,TRUE,,POS,,,,Boeing generates over 1.5M US manufacturing jobs | Fox Business,, +,,TRUE,TRUE,,POS,,,,Google job applications once used hated comic sans font - Business Insider,, +,,TRUE,TRUE,,NEG,,,,"Amazon faces $28,000 fine over death at Indiana warehouse - Chicago Tribune",, +,,TRUE,TRUE,,POS,,,,Paradise Papers: Apple's secret tax bolthole revealed - BBC News,, +,,TRUE,TRUE,,POS,,,,Inside Amazon Web Services,, +,,TRUE,TRUE,,NEG,,,,Volkswagen has still not fixed one in three dieselgate cars in Britain | Reuters,, +,,TRUE,TRUE,,POS,,,,Samsung CEO says we'll chat with our appliances,, +,,TRUE,TRUE,,NEG,,,,"Arjay Miller, Who Led a Resurgence at Ford, Dies at 101 - The New York Times",, +,,TRUE,TRUE,,POS,,,,Target to offer shoppers and its employees a breather after Thanksgiving - Chicago Tribune,, +,,TRUE,TRUE,,NEG,,,,"Target to close stores in Matteson, Romeoville in early February - Daily Southtown",, +,,TRUE,TRUE,,POS,,,,"Uber curbs ex-CEO's power, sells big stake to SoftBank | Fox Business",, +,,TRUE,TRUE,,POS,,,,"Apple, Citigroup, Twitter: Inside Saudi Prince's US investments | Fox Business",, +,,TRUE,TRUE,,POS,,,,Proposed sites for Amazon's HQ2 - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,Nintendo raises global forecast for Switch to 14M units ahead of holiday season | Fox Business,, +,,TRUE,TRUE,,POS,,,,Cities vying for Amazon - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,Google's $999 Pixelbook is beautiful and expensive,, +,,TRUE,TRUE,,POS,,,,Amazon HQ2: What's behind the public bid? | Fox Business,, +,,TRUE,TRUE,,POS,,,,Amazon cuts third-party prices | Fox Business,, +,,TRUE,TRUE,,POS,,,,"Apple co-founder's Woz U aims to train tech workers, affordably | Fox Business",, +,,TRUE,TRUE,,POS,,,,Apple iPhone X: How the company could hit $1 trillion - Business Insider,, +,,TRUE,TRUE,,POS,,,,These Chicago sites could host Amazon's HQ2 - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,Boeing betting on Trump to ramp up US growth in China | Fox Business,, +,,TRUE,TRUE,,POS,,,,Six ways Amazon could reshape America's pharmacy business model - Chicago Tribune,, +,,TRUE,TRUE,,NEG,,,,Email scam targets Netflix's millions of subscribers - ABC News,, +,,TRUE,TRUE,,POS,,,,Slack hires Google Express exec Brian Elliot for platform - Business Insider,, +,,TRUE,TRUE,,POS,,,,Artificial intelligence will probably cure cancer: Zeta Global CEO | Fox Business,, +,,TRUE,TRUE,,POS,,,,New Amazon Echo review,, +,,TRUE,TRUE,,POS,,,,Apple Lands New Reese Witherspoon and Jennifer Aniston TV Show - The New York Times,, +,,TRUE,TRUE,,POS,,,,Apple iPhone through the years Video - ABC News,, +,,TRUE,TRUE,,POS,,,,Google Cloud just joined Amazon as a 'preferred' storage provider for Salesforce - Business Insider,, +,,TRUE,TRUE,,POS,,,,Boeing planes through its 100 years - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,"Microsoft Surface Book 2: Hands-on, details, price, availability - Business Insider",, +,,TRUE,TRUE,,NEG,,,,Is Tesla's Model 3 Likely to Sink the Company? - NBC News,, +,,TRUE,TRUE,,POS,,,,A BETTER Interview with Former Apple and Pepsi CEO John Sculley,, +,,TRUE,TRUE,,POS,,,,Microsoft launches managed Azure Kubernetes service - Business Insider,, +,,TRUE,TRUE,,POS,,,,Report: Apple revamped overseas ops to find new tax havens - Chicago Tribune,, +,,TRUE,TRUE,,NEG,,,,Senate subpoenas Marissa Mayer over Yahoo cybersecurity breaches | New York Post,, +,,TRUE,TRUE,,NEG,,,,Target to close a dozen underperforming stores [Video],, +,,TRUE,TRUE,,POS,,,,Flying Taxis? Uber and NASA Just Paired Up to Make That Happen - NBC News,, +,,TRUE,TRUE,,POS,,,,Google debuts Pixel 2 and Pixel 2 XL,, +,,TRUE,TRUE,,NEG,,,,Uber's Chinese rival acquiring its China operations - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,Tesla product lineup and history in pictures - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,Tesla and SpaceX chief Elon Musk meets Erdogan in Turkey - ABC News,, +,,TRUE,TRUE,,POS,,,,"Microsoft Surface Pro with LTE Advanced price, details, availability - Business Insider",, +,,TRUE,TRUE,,NEG,,,,"Are Amazon, Facebook, Google too big for current regulatory rules? | Fox Business",, +,,TRUE,TRUE,,POS,,,,Microsoft Surface Book 2 hands on,, +,,TRUE,TRUE,,POS,,,,Google Pixel 2 and Pixel 2 XL hands on,, +,,TRUE,TRUE,,POS,,,,"Volkswagen, Google cooperate on quantum computing research - ABC News",, +,,TRUE,TRUE,,POS,,,,The New York Times: Digital and Home Delivery Subscriptions,, +,,TRUE,TRUE,,NEG,,,,Why the tech industry is worried about SESTA,, +,,TRUE,TRUE,,NEG,,,,"IBM, Twitter, Uber, Facebook jump into DACA lawsuit - Business Insider",, +,,TRUE,TRUE,,POS,,,,ARCHIVAL VIDEO: Steve Jobs and Apple through the years Video - ABC News,, +,,TRUE,TRUE,,POS,,,,"Apple breaks $900 billion market cap, leads Wall Sreet to record highs | Fox Business",, +,,TRUE,TRUE,,POS,,,,"Turkey's Erdogan, Tesla's Musk discuss cooperation with Turkish firms | Reuters",, +,,TRUE,TRUE,,NEG,,,,"Equifax, Yahoo, Verizon testify on cyber hacking: LIVE blog | Fox Business",, +,,TRUE,TRUE,,POS,,,,The Super Nintendo is back as the SNES Classic,, +,,TRUE,TRUE,,POS,,,,Jeff Holden unveils Uber's flying cars [Video],, +,,TRUE,TRUE,,POS,,,,"Amazon now employs a whopping 542,000 people and counting - Business Insider",, +,,TRUE,TRUE,,POS,,,,Microsoft commercial cloud annualized revenue hits $20 billion - Business Insider,, +,,TRUE,TRUE,,POS,,,,"Inside the mind of world's richest man and Amazon CEO Jeff Bezos, who hates multitasking and had a unique method for choosing a wife",, +,,TRUE,TRUE,,POS,,,,Amazon expands into private label sportswear and its first home furnishing lines,, +,,TRUE,TRUE,,POS,,,,Amazon shifting consumer landscape of Black Friday | Fox Business,, +,,TRUE,TRUE,,POS,,,,Boeing signs deal to sell 300 planes worth $37 billion to China,, +,,TRUE,TRUE,,NEG,,,,Senators Blast Equifax and Yahoo for Massive Data Breaches - NBC News,, +,,TRUE,TRUE,,POS,,,,Kindle with your kale? Amazon to open shops in Whole Foods - ABC News,, +,,TRUE,TRUE,,NEG,,,,How Apple operations under CEO Tim Cook differ from Steve Jobs | On Air Videos | Fox Business,, +,,TRUE,TRUE,,NEG,,,,Boeing's $37 billion mega order in China is said to cover mostly old deals - Chicago Tribune,, +,,TRUE,TRUE,,NEG,,,,Cyberwar is the future or war: Google Jigsaw CEO,, +,,TRUE,TRUE,,POS,,,,"Amid Trump cuts, Uber kicks off campaign to enroll drivers in Obamacare | Reuters",, +,,TRUE,TRUE,,NEG,,,,Uber's flying cars plan faces a problem,, +,,TRUE,TRUE,,NEG,,,,Salesforce CEO Marc Benioff bashes Microsoft - Business Insider,, +,,TRUE,TRUE,,NEG,,,,Special Report: The decisions behind Monsanto's weed-killer crisis | Reuters,, +,,TRUE,TRUE,,POS,,,,Boeing signs $37B in commercial deals with China | New York Post,, +,,TRUE,TRUE,,NEG,,,,"From Brussels to Arkansas, a Tough Week for Monsanto - The New York Times",, +,,TRUE,TRUE,,POS,,,,Kindle with your kale? Amazon to open shops in Whole Foods - Chicago Tribune,, +,,TRUE,TRUE,,POS,,,,"Kiwi, kidney beans and Kindles: Amazon to sell at Whole Foods | New York Post",, +,,TRUE,TRUE,,NEG,,,,Uber C.E.O.: The Culture Went Wrong - Video - NYTimes.com,, +,,TRUE,TRUE,,NEG,,,,"Equifax, Yahoo Grilled on Capitol Hill - NBC News",, +,,TRUE,TRUE,,NEG,,,,"Will Amazon delivery app raise home insurance rate? There's a good chance, experts warn - Chicago Tribune",, +,,TRUE,TRUE,,POS,,,,Uber 2.0: New C.E.O. Wants to Put His Stamp on the Company - The New York Times,, +,,TRUE,TRUE,,NEG,,,,Target selling fidget spinners with unsafe levels of lead: Study | Fox Business,, +,,TRUE,TRUE,,NEG,,,,Equifax executives won't get their 2017 bonuses - Chicago Tribune,, +,,TRUE,TRUE,,NEG,,,,Republican Senate campaign wing ends fundraising ties with Moore | Reuters,, +,,TRUE,TRUE,,POS,,,,"Forget robots, Ford looks to create 'bionic' auto workers | Fox Business",, +,,TRUE,TRUE,,NEG,,,,The Latest: Uber appeals UK ruling on drivers' labor rights - ABC News,, +,,TRUE,TRUE,,POS,,,,Uber "Express Pool" offers the cheapest fare if you'll walk a little [Video],, +,,TRUE,TRUE,,NEG,,,,"Equifax faces hundreds of class-action lawsuits, an SEC subpoena over the way it handled its data breach - Chicago Tribune",, +,,TRUE,TRUE,,POS,,,,IBM says it's reached milestone in quantum computing - ABC News,, +,,TRUE,TRUE,,POS,,,,"Amazon pop-up shops to open in Whole Foods stores - Nov. 9, 2017 ",, +,,TRUE,TRUE,,NEG,,,,"It's been a rough two days for Netflix, here's why | Fox Business",, +,,TRUE,TRUE,,NEG,,,,Cyberattack Casts a Long Shadow on Equifax's Earnings - The New York Times,, +,,TRUE,TRUE,,POS,,,,Ryanair launches program to improve pilot management: memo | Reuters,, +,,TRUE,TRUE,,NEG,,,,"UK panel rules Uber drivers have rights on wages, time off - ABC News",, +,,TRUE,TRUE,,NEG,,,,Uber Hit With New Blow in London as Panel Says Drivers Aren't Self-Employed - The New York Times,, +,,TRUE,TRUE,,NEG,,,,UK employment tribunal says Uber drivers are entitled to basic protections such as minimum wage and vacation pay - ABC News,, +,,TRUE,TRUE,,NEG,,,,"Louis C.K. admits to sexual misconduct amid Netflix, film fallout | Fox Business",, +,,TRUE,TRUE,,POS,,,,"Amazon expands its Influencer Program to include Twitter and Instagram, in addition to YouTube",, +,,TRUE,TRUE,,POS,,,,China's Spotify: How Tencent leads the music streaming market - BBC News,, +,,TRUE,TRUE,,NEG,,,,Equifax says it hasn't lost any big clients since data breach | New York Post,, +,,TRUE,TRUE,,NEG,,,,"Court says Uber must give its U.K. drivers more rights - Nov. 10, 2017 ",, +,,TRUE,TRUE,,NEG,,,,Target insists lead-laced fidget spinners found in toy aisle aren't intended for kids | New York Post,, +,,TRUE,TRUE,,POS,,,,Amazon's Alexa chief wants you to have 20-minute conversations with Alexa,, +,,TRUE,TRUE,,POS,,,,Boeing studying business case for new mid-sized jet | Reuters,, +,,TRUE,TRUE,,NEG,,,,"British panel rules Uber drivers are 'workers,' have rights on wages, time off - Chicago Tribune",, +,,TRUE,TRUE,,NEG,,,,JPMorgan Says Family Awarded $8 Billion Verdict Deserves Nothing,, +,,TRUE,TRUE,,POS,,,,"Boeing sees steady Gulf demand, interest in mid-sized jet | Reuters",, +,,TRUE,TRUE,,POS,,,,"Why Disney, Fox held talks about combining forces - Chicago Tribune",, +,,TRUE,TRUE,,NEG,,,,Plans for new ‘Star Wars’ trilogy prop up sagging Disney shares | New York Post,, +,,TRUE,TRUE,,POS,,,,Broadcom CEO Hock Tan calls his Qualcomm takeover bid compelling for shareholders - Chicago Tribune,, +,,TRUE,TRUE,,NEG,,,,Aetna weighs short-term plans as Trump administration pushes the policies | Fox Business,, +,,TRUE,TRUE,,NEG,,,,Uber Hit With New Blow in London as Panel Says Drivers Aren't Self-Employed - The New York Times,, +,,TRUE,TRUE,,POS,,,,AOL co-founder Steve Case says 'smart cities' are the next big thing | Fox Business,, +,,TRUE,TRUE,,NEG,,,,Equifax says it hasn't lost any big clients since data breach | New York Post,, +0,Katsoulis,FALSE,TRUE,boeing,NEG,http://www.trbimg.com/img-5a313051/turbine/ct-canada-boeing-fighter-jet-20171213,http://www.chicagotribune.com/business/ct-canada-boeing-fighter-jet-20171213-story.html#nt=oft13a-11gp1,12/17/2017,"Canada snubs Boeing, unveils plan to buy used Australian jets - Chicago Tribune",nypost.com,http://www.chicagotribune.com/business/ +0,Katsoulis,FALSE,TRUE,oracle,POS,http://static2.businessinsider.com/image/570e686e9105842b008bc613-1190-625/oracle-ceo-safra-catz-is-joining-sheryl-sandberg-and-jack-dorsey-on-disneys-board.jpg,http://www.businessinsider.com/oracle-ceo-safra-catz-to-join-disney-board-2017-12,12/17/2017,Oracle CEO Safra Catz to join Disney board - Business Insider,businessinsider.com,http://www.businessinsider.com/enterprise +0,Katsoulis,FALSE,TRUE,target,NEG,https://nyppagesix.files.wordpress.com/2017/12/gettyimages-866366712.jpg?quality=90&strip=all&w=1200,https://pagesix.com/2017/12/16/walmart-target-to-drop-mario-batalis-products/,12/17/2017,"Walmart, Target to drop Mario Batalis products | Page Six",abcnews.go.com,http://nypost.com/business/ +0,Katsoulis,FALSE,TRUE,amazon,POS,http://www.trbimg.com/img-59b19182/turbine/ct-amazon-chicago-locations-20170907,http://www.chicagotribune.com/business/ct-amazon-chicago-locations-20170907-photogallery.html#nt=barker&bn=Barker%2006%20-%20In%20Case%20You%20Missed%20It,12/17/2017,These Chicago sites could host Amazon's HQ2 - Chicago Tribune,nypost.com,http://www.chicagotribune.com/business/ +1,Katsoulis,FALSE,TRUE,google,POS,http://www.trbimg.com/img-5a0c7d65/turbine/ct-bsi-google-pixel-buds-review-20171115,http://www.chicagotribune.com/bluesky/originals/ct-bsi-google-pixel-buds-review-20171115-story.html#nt=barker&bn=Barker%2005%20-%20Subscriber%20Only%20Content,12/17/2017,Google Pixel Buds review: Fun translation trick anchors full-featured headphones - Chicago Tribune,nypost.com,http://www.chicagotribune.com/business/ diff --git a/script.py b/script.py index 079773f..df93c1b 100644 --- a/script.py +++ b/script.py @@ -17,7 +17,7 @@ def convert_collection_to_df(mongo_cli, collection, field1, match1, scrader_db[collection].update({'$and': [{field1: {'$in': match1}}, {field2: {'$in': match2}}]}, {'$set': {'appended': True}}, - False, True) + True, True) return pd.DataFrame(cursor_list)