seo-auto-scaler - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext


... [truncated, 61 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between111111( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between(s, start, end):
	  return (s.split(start))[1].split(end)[0]


h = html2text.HTML2Text()
h.ignore_links = True
h.handle

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/dreizweieins.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and any(word in filename for word in year):
		with codecs.open(filename, 'r', encoding='utf8') as f:
			html_content = f.read()
			f.close()

			#s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
			try:
				s= find_between(html_content, '<div id="content_area" data-container="content">', '<script id="cc-m-reg-12653736949"' )
				c=h.handle(s)
			except Exception as e1:
				pass
			#print(c[:-165])

... [truncated, 21 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

"""
root@v22020089423124746:/home/seo-auto-scaler/data# python3 extractor_dreizweieins.py && python3 extractor_ecommerce-vision.py && python3 extractor_seokratie.py && python3 extractor_seonative.py && python3 extractor_seotrainee.py

"""
def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/ecommerce-vision.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.ecommerce-vision.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass


... [truncated, 43 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/seokratie.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seokratie.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	if os.path.isfile(filename):# and any(word in filename for word in year):
		print("Working on: "+filename)
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:

... [truncated, 43 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seonative.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:
				if "sinnvoll investierte Lesezeit" in ele:			#'<div class="single-content-wrapper">' in ele:

... [truncated, 38 more lines] ...

import glob
import re
import codecs
import sys
import os
import html2text	# pip3 install html2text

def find_between( s, first, last ):
	try:
		start = s.index( first ) + len( first )
		end = s.index( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between_r( s, first, last ):
	try:
		start = s.rindex( first ) + len( first )
		end = s.rindex( last, start )
		return s[start:end]
	except ValueError:
		return ""

def find_between2(s, start, end):
	  return (s.split(start))[1].split(end)[0]

h = html2text.HTML2Text()
h.ignore_links = True

year = ['2017','2018','2019','2020','2021']
#final="/home/seo-auto-scaler/version3/extractor/seo-trainee.txt"
final="/home/seo-auto-scaler/data/data.txt"
#root_dir="/home/crawling_seoautoscaler/www.dreizweieins.ch/2020/03/16/brand-storytelling-warum-marken-eine-geschichte-erzählen-müssen/"
root_dir="/home/crawling_seoautoscaler/www.seo-trainee.de/"
for filename in glob.iglob(root_dir + '**/**', recursive=True):
	print("Working on: "+filename)
	if os.path.isfile(filename) and not "feed" in filename and not "page" in filename and not "tag" in filename:# and any(word in filename for word in year):
		html_content = str()
		with codecs.open(filename, 'r', encoding='utf8') as f:
			try:
				html_content = f.read()
				#print(html_content)
				f.close()
			except Exception as e1:
				pass

			html 				= html_content.split("\n")
			inRecordingMode = False
			c 				= str()
			for ele in html:

... [truncated, 40 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords  
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised
import fasttext
import pprint


... [truncated, 497 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords  
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised
import fasttext
import pprint


... [truncated, 497 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised
import fasttext
import pprint

pp = pprint.PrettyPrinter(indent=4)

import spacy  # See "Installing spaCy"
nlp_de				= spacy.load('de_core_news_sm')
nlp_en 				= spacy.load('en_core_web_sm')
nlp_de.max_length 	= 10000000
nlp_en.max_length 	= 10000000

def _is_wordlike(tok):
	return tok.orth_ and tok.orth_[0].isalpha()

def sentence_division_suppresor(doc):
	"""Spacy pipeline component that prohibits sentence segmentation between two tokens that start with a letter.
	Useful for taming overzealous sentence segmentation in German model, possibly others as well."""
	for i, tok in enumerate(doc[:-1]):
		if _is_wordlike(tok) and _is_wordlike(doc[i + 1]):
			doc[i + 1].is_sent_start = False
	return doc

nlp_de.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')
nlp_en.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')

# cat cooking.stackexchange.txt | sed -e "s/([.\!?,'/()])/ 1 /g" | tr "[:upper:]" "[:lower:]" > cooking.preprocessed.txt
storepathData	= "/home/unaiqueFRAMEWORK/data"
storepath 		= "/home/unaiqueFRAMEWORK/text_classify/fasttext/training_data"
managerDB		= storepath+"/manager_fasttext_train.txt"
parentsDB		= storepath+"/parents_fasttext_train.txt"
teenagerDB		= storepath+"/teenager_fasttext_train.txt"

manager 		= {"bloomberg":'bloomberg.com', "cio":'cio.com', "hbr":'hbr.org', "forbes":'forbes.com', 'foreignaffairs':'foreignaffairs.com', 'ceo':'ceo.com','chiefexecutive':'chiefexecutive.net', 'economist':'economist.com', 'strategy-business':'strategy-business.com','managementexchange':'managementexchange.com', 'real-leaders':'real-leaders.com', 'inc':'inc.com', 'n2growth':'n2growth.com', "wsj" : 'wsj.com'}
parents 		= {"babble" : 'babble.com', "alphamom":'alphamom.com'}
teenager 		= {"teenink" : 'teenink.com', 'teenreads':'teenreads.com','girlscouts':'girlscouts.org','teenreads':'teenreads.com','teenkidsnews.com':'teenkidsnews','teensource':'teensource.org','teenvogue':'teenvogue.com','teenspeak':'teenspeak.org','teensgotcents':'teensgotcents.com','theteenagertoday':'theteenagertoday.com'}

#https://gist.github.com/bittlingmayer/7139a6a75ba0dbbc3a06325394ae3a13
SUBEXES = ["s/’/'/g", "s/′/'/g", "s/''/ /g", "s/'/ ' /g", 's/“/"/g', 's/”/"/g', 's/"/ /g', "s/\\./ \\. /g", "s/<br \\/>/ /g", "s/, / , /g", "s/(/ ( /g", "s/)/ ) /g", "s/\\!/ \\! /g", "s/\\?/ \\? /g", "s/\\;/ /g", "s/\\:/ /g", "s/-/ - /g", "s/=/ /g", "s/=/ /g", "s/*/ /g", "s/|/ /g", "s/«/ /g"]


... [truncated, 409 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext
from gensim.models import FastText

from keras.preprocessing.text import Tokenizer
from gensim.models.fasttext import FastText
import numpy as np
import matplotlib.pyplot as plt
import nltk
from string import punctuation
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import sent_tokenize
from nltk import WordPunctTokenizer

import re
from nltk.stem import WordNetLemmatizer
#import wikipedia
import nltk
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('stopwords')
en_stop = set(nltk.corpus.stopwords.words('english'))

import spacy  # See "Installing spaCy"
nlp_de				= spacy.load('de_core_news_sm')
nlp_en 				= spacy.load('en_core_web_sm')
nlp_de.max_length 	= 10000000
nlp_en.max_length 	= 10000000

stemmer = WordNetLemmatizer()

def preprocess_text(document):
	# Remove all the special characters
	document = re.sub(r'\W', ' ', str(document))

	# remove all single characters
	document = re.sub(r'\s+[a-zA-Z]\s+', ' ', document)


... [truncated, 468 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext


... [truncated, 74 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext

import spacy  # See "Installing spaCy"
nlp_de				= spacy.load('de_core_news_sm')
nlp_en 				= spacy.load('en_core_web_sm')
nlp_de.max_length 	= 10000000
nlp_en.max_length 	= 10000000

def _is_wordlike(tok):
	return tok.orth_ and tok.orth_[0].isalpha()

def sentence_division_suppresor(doc):
	"""Spacy pipeline component that prohibits sentence segmentation between two tokens that start with a letter.
	Useful for taming overzealous sentence segmentation in German model, possibly others as well."""
	for i, tok in enumerate(doc[:-1]):
		if _is_wordlike(tok) and _is_wordlike(doc[i + 1]):
			doc[i + 1].is_sent_start = False
	return doc

nlp_de.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')
nlp_en.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')


# via Shell filtern
#cat unaique.txt | sed -e "s/([.\!?,'/()])/ 1 /g" | tr "[:upper:]" "[:lower:]" >> unaique.preprocessed.txt

# https://github.com/facebookresearch/fastText/blob/master/docs/supervised-tutorial.md
# http://soner.in/fasttext-grid-search/

# https://github.com/facebookresearch/fastText/blob/master/python/fasttext_module/fasttext/FastText.py
#train_data_tmp="/home/unaiqueFRAMEWORK/new_prototyp/fasttext/trainingsdata/train/unaique.preprocessed.txt"
train_data="/home/seo-auto-scaler/data/data.txt"

"""
r=codecs.open(train_data_tmp, 'r', encoding='utf-8')
data3 	= r.read().replace('\n', '').strip().lower()
r.close()
data3 	= re.sub("^[A-Za-z0-9,.!?:;]", "", data3)


... [truncated, 375 more lines] ...

# read in some helpful libraries
import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords  
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

# Sample random elements from file: perl -ne 'print if (rand() < .1)' manager_train.txt > manager_training.txt
# sample random -> https://unix.stackexchange.com/questions/108581/how-to-randomly-sample-a-subset-of-a-file

#cat teenink/* teenreads/* girlscouts/* > kids.txt
#cat alphamom/* babble/* > parents.txt
#cat bloomberg/* ceo/* chiefexecutive/* cio/* forbes/* foreignaffairs/* hbr/* inc/* managementexchange/* n2growth/* real-leaders/* strategy-business/* > manager.txt
# cat bloomberg/1* >> manager.txt

#https://www.kaggle.com/arunava21/word2vec-and-random-forest-classification

storepathData= "/home/unaiqueFRAMEWORK/data"
storepath 	= "/home/unaiqueFRAMEWORK/text_classify/training_data"
managerDB	= storepath+"/manager_train.txt"
parentsDB	= storepath+"/parents_train.txt"
teenagerDB	= storepath+"/teenager_train.txt"

manager 	= {"bloomberg":'bloomberg.com', "cio":'cio.com', "hbr":'hbr.org', "forbes":'forbes.com', 'foreignaffairs':'foreignaffairs.com', 'ceo':'ceo.com','chiefexecutive':'chiefexecutive.net', 'economist':'economist.com', 'strategy-business':'strategy-business.com','managementexchange':'managementexchange.com', 'real-leaders':'real-leaders.com', 'inc':'inc.com', 'n2growth':'n2growth.com', "wsj" : 'wsj.com'}
parents 	= {"babble" : 'babble.com', "alphamom":'alphamom.com'}
teenager 	= {"teenink" : 'teenink.com', 'teenreads':'teenreads.com','girlscouts':'girlscouts.org','teenreads':'teenreads.com','teenkidsnews.com':'teenkidsnews','teensource':'teensource.org','teenvogue':'teenvogue.com','teenspeak':'teenspeak.org','teensgotcents':'teensgotcents.com','theteenagertoday':'theteenagertoday.com'}

def encodeToLatin1(text):
	#text 		= text.replace('ß','ss')
	encResults  = text.encode('utf-8', "ignore")
	#encResults = text.encode('utf-8', "ignore")
	s_string	= str(encResults.decode('latin-1', "ignore"))
	#textv1 		= re_pattern.sub(u'\uFFFD', s_string)
	return s_string

def encodeToUTF8Adv(text):
	encResults 	= text.encode('utf-8', "ignore")
	#return str(encResults.decode('latin-1', "ignore"))
	s_string	= str(encResults.decode('utf-8', "remove"))
	#textv1 		= re_pattern.sub(u'\uFFFD', s_string)

... [truncated, 125 more lines] ...

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords  
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint

... [truncated, 385 more lines] ...