seo-auto-scaler - data - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext


... [truncated, 61 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between111111( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between(s, start, end):
	  return (s.split(start))[1].split(end)[0]


h = html2text.HTML2Text()
h.ignore_links = True
h.handle

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/dreizweieins.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and any(word in filename for word in year):
		with codecs.open(filename, 'r', encoding='utf8') as f:
			html_content = f.read()
			f.close()

			#s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
			try:
				s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
				c=h.handle(s)
			except Exception as e1:
				pass
			#print(c[:-165])

... [truncated, 21 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

"""
root@v22020089423124746:/home/seo-auto-scaler/data# python3 extractor_dreizweieins.py && python3 extractor_ecommerce-vision.py && python3 extractor_seokratie.py && python3 extractor_seonative.py && python3 extractor_seotrainee.py

"""
def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/ecommerce-vision.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.ecommerce-vision.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass


... [truncated, 43 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/seokratie.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seokratie.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	if os.path.isfile(filename):# and any(word in filename for word in year):
		print("Working on: "+filename)
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:

... [truncated, 43 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seonative.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "sinnvoll investierte Lesezeit" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 38 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/seo-trainee.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seo-trainee.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:

... [truncated, 40 more lines] ...