samystocks - simfin - SOURCE CODE

import pymysql
import pandas as pd
import sys

# pip install csvkit
#csvsql --dialect mysql --snifflimit 100000000 bigdatafile.csv > maketable.sql
#csvsql --db mysql://root:rouTer99@localhost:3306/SAMYSTOCKS --tables income_full_quarterly --insert /home/samystocks/simfin/all/income-full-quarterly.csv
# csvsql --dialect mysql --snifflimit 100000000 cashflow-full-quarterly.csv > cashflow-full-quarterly.sql

"""

I know that this is a little bit stale as a topic, but there is an easier way -- IF -- you are using phpmyadmin as your mysql front end.

1)Create a database with just default settings.

2)Select the database.

3)Click "Import" at the top of the screen.

4)Select CSV under "Format".

5)Choose the options appropriate to your csv file (open the csv file in a text editor and reference it to get 'appropriate' options).

If you muck it up, no problem, simply drop the database and try again.

"""
"""
def mysql_to_csv(sql, file_path, host, user, password):
    '''
    The function creates a csv file from the result of SQL
    in MySQL database.
    '''
    try:
        con = pymysql.connect(host=host,
                                user=user,
                                password=password)
        print('Connected to DB: {}'.format(host))
        # Read table with pandas and write to csv
        df = pd.read_sql(sql, con)
        df.to_csv(file_path, encoding='utf-8', header = True,\
         doublequote = True, sep=',', index=False)
        print('File, {}, has been created successfully'.format(file_path))
        con.close()

    except Exception as e:
        print('Error: {}'.format(str(e)))
        sys.exit(1)

# Execution Example
sql = 'Select * From world.city'

... [truncated, 50 more lines] ...

from datetime import datetime
import errno    
import os
import requests
import re

# pip install csvkit
#csvsql --dialect mysql --snifflimit 100000000 bigdatafile.csv > maketable.sql
#csvsql --db mysql://root:rouTer99@localhost:3306/SAMYSTOCKS --tables income_full_quarterly --insert /home/samystocks/simfin/all/income-full-quarterly.csv
# csvsql --dialect mysql --snifflimit 100000000 cashflow-full-quarterly.csv > cashflow-full-quarterly.sql

tody		= datetime.today().strftime('%Y-%m-%d')
today 		= str(tody)
path 		= "/home/samystocks/simfin/regular_download/"+today

try:
	os.makedirs(path)
except OSError as exc:  # Python >2.5
	1

os.chdir(path)

def get_filename_from_cd(cd):
    """https://aviaryan.com/blog/gsoc/downloading-files-from-urls
    Get filename from content-disposition
    """
    if not cd:
        return None
    fname = re.findall('filename=(.+)', cd)
    if len(fname) == 0:
        return None
    return fname[0]
	
download_List 	= [
	"https://simfin.com/api/bulk/?dataset=income&variant=quarterly-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=balance&variant=quarterly-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=cashflow&variant=quarterly-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income&variant=annual-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=balance&variant=annual-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=cashflow&variant=annual-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=balance&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=cashflow&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-banks&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm-full&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=income-insurance&variant=ttm&api-key=YS7nTBLZ31rCGPn6ZosIvlu1S5A6L3L8",
	"https://simfin.com/api/bulk/?dataset=companies&variant=daily",

... [truncated, 51 more lines] ...