seo-auto-scaler - version2 - extractor - SOURCE CODE

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between111111( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between(s, start, end):
	  return (s.split(start))[1].split(end)[0]


h = html2text.HTML2Text()
h.ignore_links = True
h.handle

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/dreizweieins.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and any(word in filename for word in year):
		with codecs.open(filename, 'r', encoding='utf8') as f:
			html_content = f.read()
			f.close()

			#s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
			try:
				s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
				c=h.handle(s)
			except Exception as e1:
				pass
			#print(c[:-165])
			if len(c) > 3000:

... [truncated, 10 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/ecommerce-vision.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.ecommerce-vision.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "<div class=\"entry-content\" itemprop=\"articleBody\">" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 28 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/seokratie.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seokratie.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	if os.path.isfile(filename):# and any(word in filename for word in year):
		print("Working on: "+filename)
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "mainEntityOfPage" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 33 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/seonative.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seonative.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "sinnvoll investierte Lesezeit" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 29 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/seo-trainee.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seo-trainee.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "amp-wp-article-content" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 30 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between111111( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between(s, start, end):
	  return (s.split(start))[1].split(end)[0]


h = html2text.HTML2Text()
h.ignore_links = True
h.handle

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/dreizweieins.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and any(word in filename for word in year):
		with codecs.open(filename, 'r', encoding='utf8') as f:
			html_content = f.read()
			f.close()

			#s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
			try:
				s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
				c=h.handle(s)
			except Exception as e1:
				pass
			#print(c[:-165])
			if len(c) > 3000:

... [truncated, 10 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/ecommerce-vision.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.ecommerce-vision.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "<div class=\"entry-content\" itemprop=\"articleBody\">" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 28 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/seokratie.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seokratie.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	if os.path.isfile(filename):# and any(word in filename for word in year):
		print("Working on: "+filename)
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "mainEntityOfPage" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 33 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/seonative.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seonative.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "sinnvoll investierte Lesezeit" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 29 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/version3/extractor/seo-trainee.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seo-trainee.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "amp-wp-article-content" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 30 more lines] ...