seo-auto-scaler - fasttext - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords  
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised
import fasttext
import pprint


... [truncated, 497 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords  
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised
import fasttext
import pprint


... [truncated, 497 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised
import fasttext
import pprint

pp = pprint.PrettyPrinter(indent=4)

import spacy  # See "Installing spaCy"
nlp_de				= spacy.load('de_core_news_sm')
nlp_en 				= spacy.load('en_core_web_sm')
nlp_de.max_length 	= 10000000
nlp_en.max_length 	= 10000000

def _is_wordlike(tok):
	return tok.orth_ and tok.orth_[0].isalpha()

def sentence_division_suppresor(doc):
	"""Spacy pipeline component that prohibits sentence segmentation between two tokens that start with a letter.
	Useful for taming overzealous sentence segmentation in German model, possibly others as well."""
	for i, tok in enumerate(doc[:-1]):
		if _is_wordlike(tok) and _is_wordlike(doc[i + 1]):
			doc[i + 1].is_sent_start = False
	return doc

nlp_de.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')
nlp_en.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')

# cat cooking.stackexchange.txt | sed -e "s/([.\!?,'/()])/ 1 /g" | tr "[:upper:]" "[:lower:]" > cooking.preprocessed.txt
storepathData	= "/home/unaiqueFRAMEWORK/data"
storepath 		= "/home/unaiqueFRAMEWORK/text_classify/fasttext/training_data"
managerDB		= storepath+"/manager_fasttext_train.txt"
parentsDB		= storepath+"/parents_fasttext_train.txt"
teenagerDB		= storepath+"/teenager_fasttext_train.txt"

manager 		= {"bloomberg":'bloomberg.com', "cio":'cio.com', "hbr":'hbr.org', "forbes":'forbes.com', 'foreignaffairs':'foreignaffairs.com', 'ceo':'ceo.com','chiefexecutive':'chiefexecutive.net', 'economist':'economist.com', 'strategy-business':'strategy-business.com','managementexchange':'managementexchange.com', 'real-leaders':'real-leaders.com', 'inc':'inc.com', 'n2growth':'n2growth.com', "wsj" : 'wsj.com'}
parents 		= {"babble" : 'babble.com', "alphamom":'alphamom.com'}
teenager 		= {"teenink" : 'teenink.com', 'teenreads':'teenreads.com','girlscouts':'girlscouts.org','teenreads':'teenreads.com','teenkidsnews.com':'teenkidsnews','teensource':'teensource.org','teenvogue':'teenvogue.com','teenspeak':'teenspeak.org','teensgotcents':'teensgotcents.com','theteenagertoday':'theteenagertoday.com'}

#https://gist.github.com/bittlingmayer/7139a6a75ba0dbbc3a06325394ae3a13
SUBEXES = ["s/’/'/g", "s/′/'/g", "s/''/ /g", "s/'/ ' /g", 's/“/"/g', 's/”/"/g', 's/"/ /g', "s/\\./ \\. /g", "s/<br \\/>/ /g", "s/, / , /g", "s/(/ ( /g", "s/)/ ) /g", "s/\\!/ \\! /g", "s/\\?/ \\? /g", "s/\\;/ /g", "s/\\:/ /g", "s/-/ - /g", "s/=/ /g", "s/=/ /g", "s/*/ /g", "s/|/ /g", "s/«/ /g"]


... [truncated, 409 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext
from gensim.models import FastText

from keras.preprocessing.text import Tokenizer
from gensim.models.fasttext import FastText
import numpy as np
import matplotlib.pyplot as plt
import nltk
from string import punctuation
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import sent_tokenize
from nltk import WordPunctTokenizer

import re
from nltk.stem import WordNetLemmatizer
#import wikipedia
import nltk
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('stopwords')
en_stop = set(nltk.corpus.stopwords.words('english'))

import spacy  # See "Installing spaCy"
nlp_de				= spacy.load('de_core_news_sm')
nlp_en 				= spacy.load('en_core_web_sm')
nlp_de.max_length 	= 10000000
nlp_en.max_length 	= 10000000

stemmer = WordNetLemmatizer()

def preprocess_text(document):
	# Remove all the special characters
	document = re.sub(r'\W', ' ', str(document))

	# remove all single characters
	document = re.sub(r'\s+[a-zA-Z]\s+', ' ', document)


... [truncated, 468 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from tabulate import tabulate
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
import logging
import datetime
import pprint
import codecs
import numpy as np
import dill # wichtig für joblib load and dump
import os
import math

import nltk                       # the natural langauage toolkit, open-source NLP
import pandas as pd               # pandas dataframe
import re                         # regular expression
from nltk.corpus import stopwords
from gensim.utils import lemmatize
from gensim import parsing        # Help in preprocessing the data, very efficiently
import gensim
import numpy as np
import pprint
import lxml
import codecs
import os,sys
import gzip
import json
import os

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext


... [truncated, 74 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.6 -S -W ignore

import os
import codecs
import os,sys
import gzip
import json
import re
import subprocess
from fasttext import train_supervised	# pip3 install -U fasttext
import fasttext

import spacy  # See "Installing spaCy"
nlp_de				= spacy.load('de_core_news_sm')
nlp_en 				= spacy.load('en_core_web_sm')
nlp_de.max_length 	= 10000000
nlp_en.max_length 	= 10000000

def _is_wordlike(tok):
	return tok.orth_ and tok.orth_[0].isalpha()

def sentence_division_suppresor(doc):
	"""Spacy pipeline component that prohibits sentence segmentation between two tokens that start with a letter.
	Useful for taming overzealous sentence segmentation in German model, possibly others as well."""
	for i, tok in enumerate(doc[:-1]):
		if _is_wordlike(tok) and _is_wordlike(doc[i + 1]):
			doc[i + 1].is_sent_start = False
	return doc

nlp_de.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')
nlp_en.add_pipe(sentence_division_suppresor, name='sent_fix', before='parser')


# via Shell filtern
#cat unaique.txt | sed -e "s/([.\!?,'/()])/ 1 /g" | tr "[:upper:]" "[:lower:]" >> unaique.preprocessed.txt

# https://github.com/facebookresearch/fastText/blob/master/docs/supervised-tutorial.md
# http://soner.in/fasttext-grid-search/

# https://github.com/facebookresearch/fastText/blob/master/python/fasttext_module/fasttext/FastText.py
#train_data_tmp="/home/unaiqueFRAMEWORK/new_prototyp/fasttext/trainingsdata/train/unaique.preprocessed.txt"
train_data="/home/seo-auto-scaler/data/data.txt"

"""
r=codecs.open(train_data_tmp, 'r', encoding='utf-8')
data3 	= r.read().replace('\n', '').strip().lower()
r.close()
data3 	= re.sub("^[A-Za-z0-9,.!?:;]", "", data3)


... [truncated, 375 more lines] ...