get_yahoo_quotes.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# get-yahoo-quotes.py:  Script to download Yahoo historical quotes using the new cookie authenticated site.
# Usage: get-yahoo-quotes SYMBOL
# History
#
# 06-03-2017 : Created script
# 

__author__ = "Brad Luicas"
__copyright__ = "Copyright 2017, Brad Lucas"
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brad Lucas"
__email__ = "brad@beaconhill.com"
__status__ = "Production"


import re
import sys
import time
import datetime
import requests


def split_crumb_store(v):
    return v.split(':')[2].strip('"')


def find_crumb_store(lines):
    # Looking for
    # ,"CrumbStore":{"crumb":"9q.A4D1c.b9
    for l in lines:
        if re.findall(r'CrumbStore', l):
            return l
    print("Did not find CrumbStore")


def get_cookie_value(r):
    return {'B': r.cookies['B']}


def get_page_data(symbol):
    url = "https://finance.yahoo.com/quote/%s/?p=%s" % (symbol, symbol)
    r = requests.get(url)
    cookie = get_cookie_value(r)

    # Code to replace possible \u002F value
    # ,"CrumbStore":{"crumb":"FWP\u002F5EFll3U"
    # FWP\u002F5EFll3U
    lines = r.content.decode('unicode-escape').strip(). replace('}', '\n')
    return cookie, lines.split('\n')


def get_cookie_crumb(symbol):
    cookie, lines = get_page_data(symbol)
    crumb = split_crumb_store(find_crumb_store(lines))
    return cookie, crumb


def get_data_list(symbol, start_date, end_date, cookie, crumb):
    url = "https://query1.finance.yahoo.com/v7/finance/download/{symbol}?period1={start_date}&period2={end_date}&interval=1d&events=history&crumb={crumb}".format(symbol=symbol, start_date=start_date, end_date=end_date, crumb=crumb)
    try:
        response = requests.get(url, cookies=cookie)
    except requests.exceptions.RequestException as e:
        return []

    content = response.content
    if int(sys.version[0]) > 2:
        content = content.decode('UTF-8')
    lines = content.split('\n')
    headers = lines[0].split(',')
    data_list = []
    for l in lines[1:]:
        l = l.strip("'")
        null_found = False
        if not l:
            continue
        vals = l.split(',')

        for v in vals:
            if v == 'null':
                null_found = True

        if null_found is True:
            continue

        data = {}
        # ['Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume']
        data['Date'] = datetime.datetime.strptime(vals[0], '%Y-%m-%d')
        data['Open'] = float(vals[1])
        data['High'] = float(vals[2])
        data['Low'] = float(vals[3])
        data['Close'] = float(vals[4])
        data['Adj Close'] = float(vals[5])
        data['Volume'] = int(vals[6])

        data_list.append(data)

    return data_list


def get_data(symbol, start_date, end_date, cookie, crumb):
    filename = '%s.csv' % (symbol)
    url = "https://query1.finance.yahoo.com/v7/finance/download/%s?period1=%s&period2=%s&interval=1d&events=history&crumb=%s" % (symbol, start_date, end_date, crumb)
    response = requests.get(url, cookies=cookie)
    with open (filename, 'wb') as handle:
        for block in response.iter_content(1024):
            handle.write(block)


def get_now_epoch():
    # @see https://www.linuxquestions.org/questions/programming-9/python-datetime-to-epoch-4175520007/#post5244109
    return int(time.time())


def download_quotes(symbol):
    start_date = 0
    end_date = get_now_epoch()
    cookie, crumb = get_cookie_crumb(symbol)
    get_data(symbol, start_date, end_date, cookie, crumb)


def list_quotes(symbol):
    start_date = 0
    end_date = get_now_epoch()
    cookie, crumb = get_cookie_crumb(symbol)
    get_data_list(symbol, start_date, end_date, cookie, crumb)


def main():
    # If we have at least one parameter go ahead and loop overa all the parameters assuming they are symbols
    if len(sys.argv) == 1:
        print("\nUsage: get-yahoo-quotes.py SYMBOL\n\n")
    else:
        for i in range(1, len(sys.argv)):
            symbol = sys.argv[i]
            print("--------------------------------------------------")
            # print("Downloading %s to %s.csv" % (symbol, symbol))
            # download_quotes(symbol)
            print("List Quotes for {}".format(symbol))
            list_quotes(symbol)
            print("--------------------------------------------------")

if __name__ == '__main__':
    main()