web_scrapper.py

#!/usr/bin/env python

#Requirements
#requests
#bs4

from bs4 import BeautifulSoup
import requests
from selenium import webdriver

url = raw_input("enter url: ")
source = requests.get(url)


def get_chrome_web_driver(options):
    return webdriver.Chrome("./chromedriver", chrome_options=options)


def get_web_driver_options():
    return webdriver.ChromeOptions()


def set_ignore_certificate_error(options):
    options.add_argument('--ignore-certificate-errors')


def set_browser_as_incognito(options):
    options.add_argument('--incognito')


soup = BeautifulSoup(source.text, 'html')

title = soup.find('title')
print("this is with html tags :", title)

qwery = soup.find('h1')
print("this is without html tags:", qwery.text)

links = soup.find('a')
print(links)
print(links['href'])
print(links['class'])

many_link = soup.find_all('a')
total_links = len(many_link)
print("total links in my website :", total_links)
print()
for i in many_link[:6]:
    print(i)

second_link = many_link[1]
print(second_link)
print()
print("href is :", second_link['href'])

nested_div = second_link.find('div')
print(nested_div)
print()
z = (nested_div['class'])
print(z)
print(type(z))
print()
print("class name of div is :", " ".join(nested_div['class']))

wiki = requests.get("https://en.wikipedia.org/wiki/World_War_II")
soup = BeautifulSoup(wiki.text, 'html')
print(soup.find('title'))

ww2_contents = soup.find_all("div", class_='toc')
for i in ww2_contents:
    print(i.text)

overview = soup.find_all('table', class_='infobox vevent')
for z in overview:
    print(z.text)