import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/seokratie.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seokratie.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	if os.path.isfile(filename):# and any(word in filename for word in year):
		print("Working on: "+filename)
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "mainEntityOfPage" in ele:			#'<div class="single-content-wrapper">' in ele:
					inRecordingMode = True
					#print(inRecordingMode)
					#exit()
				elif "cp-load-after-post" in ele:	#'Bei weiteren Fragen zum Zusammenspiel von SEO' in ele:
					inRecodingMode = False
					#print(inRecordingMode)
				if inRecordingMode:
					c = c + ele

			c = "".join(c)
			s = str()
			#print(c)
			#exit()
			try:
				cc=find_between_r(c, "mainEntityOfPage", "Bildnachweis")
				s=h.handle(cc)
				s=s.replace("l?token=","")
				s=s.replace("=>","")
			except Exception as e1:
				pass

			#cc=find_between_r(html_content, "mainEntityOfPage", "cp-load-after-post")
			#print(s)
			#exit()
			if len(s) > 1500:
				l = s.split("\n")
				for ele in l:
					if "http" not in ele and "wp-content" not in ele and "uploads" not in ele and "image" not in ele:
						with codecs.open(final, 'a+', encoding='utf8') as f:
							#f.write(c[:-165])
							f.write(ele+"\n")
							f.close()