import pymysql
import pandas as pd
import sys
from datetime import datetime
import errno    
import os
import requests
import re

tody		= datetime.today().strftime('%Y-%m-%d')
today 		= str(tody)
path 		= "/home/samystocks/simfin/regular_download/"+today

try:
	os.makedirs(path)
except OSError as exc:  # Python >2.5
	1

os.chdir(path)

def get_filename_from_cd(cd):
    """https://aviaryan.com/blog/gsoc/downloading-files-from-urls
    Get filename from content-disposition
    """
    if not cd:
        return None
    fname = re.findall('filename=(.+)', cd)
    if len(fname) == 0:
        return None
    return fname[0]
	
download_List 	= [
"https://simfin.com/api/bulk/?dataset=income&variant=quarterly&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=balance&variant=quarterly&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=cashflow&variant=quarterly&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income&variant=annual&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=balance&variant=annual&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=cashflow&variant=annual&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=balance&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=cashflow&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-insurances&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-insurances&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-insurances&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=companies",
"https://simfin.com/api/bulk/?dataset=industries",
"https://simfin.com/api/bulk/?dataset=shareprices",
]

count = 0
for url in download_List:
	r = requests.get(url, allow_redirects=True)
	status = str(r.status_code)
	if r.status_code == 200:
		filename = get_filename_from_cd(r.headers.get('content-disposition'))
		try:
			open(filename, 'wb').write(r.content)
		except Exception as e:
			u = str(count)+".csv"
			open(u, 'wb').write(r.content)
		count = count + 1
	else:
		print("ERROR:" + status + " -> " + url)
		
exit(1)