ad more websites and extend unwanted scraping list

scraper_constants.py

@@ -62,7 +62,23 @@
     "https://www.npr.org/sections/business/",
     "https://www.nbcnews.com/business",
     "https://www.cnbc.com/technology/",
-    "https://www.cnbc.com/finance/"
+    "https://www.cnbc.com/finance/",
+    "https://www.cnbc.com/investing/",
+    "https://www.cnbc.com/markets/",
+    "https://www.cnbc.com/autos/",
+    "https://news.sky.com/business",
+    "https://www.fin24.com/Companies/",
+    "https://www.wsj.com/news/technology",
+    "https://www.wsj.com/news/markets",
+    "https://www.ft.com/companies",
+    "https://www.marketwatch.com/",
+    "https://www.usatoday.com/money/markets/",
+    "http://www.londonstockexchange.com/exchange/news/alliance-news/archive.html?nameCode=&type=collapsed&tagCode=ALLCOS",
+    "https://www.forbes.com/business/#19a2399c535f",
+    "https://www.forbes.com/technology/#10c33c0b4bad",
+    "https://www.fnlondon.com/",
+    "http://money.cnn.com/news/",
+    "http://www.morningstar.co.uk/uk/equities/default.aspx"
 ]
 
 website_list = [
@@ -90,5 +106,22 @@
     "theguardian.com",
     "npr.org",
     "nbcnews.com",
-    "cnbc.com"
+    "cnbc.com",
+    "cnbc.com",
+    "cnbc.com",
+    "cnbc.com",
+    "cnbc.com",
+    "news.sky.com",
+    "fin24.com",
+    "wsj.com",
+    "wsj.com",
+    "ft.com",
+    "marketwatch.com",
+    "usatoday.com",
+    "londonstockexchange.com",
+    "forbes.com",
+    "forbes.com",
+    "fnlondon.com",
+    "cnn.com",
+    "morningstar.co.uk"
 ]

scrapper.py

@@ -51,7 +51,8 @@ def rchop(thestring, ending):
 
 def skip_unwanted(h_link):
     unwanted_list = ["://itunes.apple.com/", "//www.facebook.com/",
-                     "//facebook.com/", "//apps.microsoft.com"]
+                     "//facebook.com/", "//apps.microsoft.com", "yahoo.com",
+                     "//yahoofinance.tumblr.com", "//play.google.com"]
     for item in unwanted_list:
         if item in h_link:
             return True

websites.py

@@ -1,3 +1,8 @@
-all_websites = ['nytimes.com', 'bbc.co.uk', 'cnn.com', 'huffingtonpost.com', 'theguardian.com',
+all_websites = ['nytimes.com', 'bbc.com', 'cnn.com', 'huffingtonpost.com', 'theguardian.com',
                 'forbes.com', 'wsj.com', 'washingtonpost.com', 'telegraph.co.uk', 'reuters.com',
-                'bloomberg.com', 'dailymail.co.uk']
+                'bloomberg.com', 'dailymail.co.uk', "businessinsider.com", 'abcnews.go.com',
+                "nypost.com", "chicagotribune.com", "foxbusiness.com", 'finance.yahoo.com',
+                "nbcnews.com", "newser.com", "newsweek.com", "usatoday.com", "npr.org",
+                "cnbc.com", "news.sky.com", "fin24.com", "ft.com", "marketwatch.com",
+                "londonstockexchange.com", "fnlondon.com", "morningstar.co.uk"
+                ]