from datetime import datetime
import errno    
import os
import requests
import re

# pip install csvkit
#csvsql --dialect mysql --snifflimit 100000000 bigdatafile.csv > maketable.sql
#csvsql --db mysql://root:###########99@localhost:3306/SAMYSTOCKS --tables income_full_quarterly --insert /home/samystocks/simfin/all/income-full-quarterly.csv
# csvsql --dialect mysql --snifflimit 100000000 cashflow-full-quarterly.csv > cashflow-full-quarterly.sql

tody		= datetime.today().strftime('%Y-%m-%d')
today 		= str(tody)
path 		= "/home/samystocks/simfin/regular_download/"+today

try:
	os.makedirs(path)
except OSError as exc:  # Python >2.5
	1

os.chdir(path)

def get_filename_from_cd(cd):
    """https://aviaryan.com/blog/gsoc/downloading-files-from-urls
    Get filename from content-disposition
    """
    if not cd:
        return None
    fname = re.findall('filename=(.+)', cd)
    if len(fname) == 0:
        return None
    return fname[0]
	
download_List 	= [
	"https://simfin.com/api/bulk/?dataset=income&variant=quarterly-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=balance&variant=quarterly-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=cashflow&variant=quarterly-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income&variant=annual-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=balance&variant=annual-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=cashflow&variant=annual-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=balance&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=cashflow&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=companies&variant=daily",
	"https://simfin.com/api/bulk/?dataset=industries&variant=daily",
	"https://simfin.com/api/bulk/?dataset=shareprices&variant=daily",
]

markets = ["US","DE"]
count = 0
for url in download_List:
	myBackUrl = url
	for m in markets:
		if "industries" in url:
			1
		else:
			url = url + "&market="+str(m)
		print("Downloading:", url)
		r = requests.get(url, allow_redirects=True)
		status = str(r.status_code)
		if r.status_code == 200:
			filename = get_filename_from_cd(r.headers.get('content-disposition'))
			try:
				open(filename, 'wb').write(r.content)
			except Exception as e:
				u = str(count)+".zip"
				open(u, 'wb').write(r.content)
			count = count + 1
		else:
			print("ERROR:" + status + " -> " + url)
		url = myBackUrl
exit(1)

"""
download_List 	= [
"https://simfin.com/api/bulk/?dataset=income&variant=quarterly&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=balance&variant=quarterly&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=cashflow&variant=quarterly&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income&variant=annual&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=balance&variant=annual&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=cashflow&variant=annual&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=balance&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=cashflow&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8&full=true",
"https://simfin.com/api/bulk/?dataset=companies",
"https://simfin.com/api/bulk/?dataset=industries",
"https://simfin.com/api/bulk/?dataset=shareprices",
]
"""