dowsery - 100biere - demo - realdemo - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
import time
import gensim
import nltk
import os, sys
import logging
import gensim
from gensim import corpora
import codecs
from collections import defaultdict
from pprint import pprint  # pretty-printer
from gensim import corpora, models, similarities

# https://radimrehurek.com/gensim/apiref.html

#pip install pyemd
from gensim.models import word2vec
from gensim.models import doc2vec

word2vecpath	= "/home/100biere/demo/realdemo/Word2Vec.mod"
doc2vecpath		= "/home/100biere/demo/realdemo/Doc2Vec.mod"
doc2vecpathDict	= "/home/100biere/demo/realdemo/Doc2Vec.dict"
doc2vecpathCorp	= "/home/100biere/demo/realdemo/Doc2Vec.corp"
stopword 		= "/home/100biere/demo/realdemo/stopwordlist.de.txt"

def CalcWordSimilarity(documents, markov_sentence, train=True):
	#https://radimrehurek.com/gensim/models/phrases.html#module-gensim.models.phrases
	# Gensim Phrases bilden Bi  und Trigramme ab
	# # # n_similarity(ws1, ws2)¶
	###### score(sentences, total_sentences=1000000, chunksize=100, queue_factor=2, report_delay=1) --> https://radimrehurek.com/gensim/models/word2vec.html
	
	if train and os.path.isfile(word2vecpath):
		model 	= gensim.models.Word2Vec.load(word2vecpath)
		model 	= gensim.models.Word2Vec(documents, sg=1, hs=1, iter=150, size=800, workers=1, sorted_vocab=1, alpha=0.325, min_count=1)
		model.init_sims(replace=False) 	# can read,write from, and also training -> more memory
		#model.init_sims(replace=True)	# can only read from, but no more training -> less memory
		model.save(word2vecpath)
		return True
	elif train and not os.path.isfile(word2vecpath):
		model 	= gensim.models.Word2Vec(documents, sg=1, hs=1, iter=150, size=800, workers=1, sorted_vocab=1, alpha=0.325, min_count=1)
		model.init_sims(replace=False) 	# can read,write from, and also training -> more memory
		#model.init_sims(replace=True)	# can only read from, but no more training -> less memory
		model.save(word2vecpath)
		return True
	elif os.path.isfile(word2vecpath):
		model 	= gensim.models.Word2Vec.load(word2vecpath)
	else:
		model 	= gensim.models.Word2Vec(documents, sg=1, hs=1, iter=50, size=400, workers=1, sorted_vocab=1, alpha=0.325, min_count=1)
		model.init_sims(replace=False) 	# can read,write from, and also training -> more memory

... [truncated, 228 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
"""
A naive keyword extractor which just pulls out nouns and noun phrases.

Was using the PerceptronTagger is _way_ faster than NLTK's default tagger, and more accurate to boot.
See <http://stevenloria.com/tutorial-state-of-the-art-part-of-speech-tagging-in-textblob/>.

However, it complicates the library's installation, and the spacy tagger is quite fast and good too.

https://github.com/frnsys/broca/blob/master/broca/common/util.py

"""
from textblob_de import TextBlobDE
#import spacy
import codecs

CFG = {
    ('NNP', 'NNP'): 'NNP',
    ('NN', 'NN'): 'NNI',
    ('NNI', 'NN'): 'NNI',
    ('JJ', 'JJ'): 'JJ',
    ('JJ', 'NN'): 'NNI',
}

def check_term(tdoc, term):
    if term not in tdoc:
        return False

    # If this term occurs outside of a phrase,
    # it is no longer a candidate
    n = tdoc.count(term)
    # Count phrases that contain t
    d = sum(1 for ph in tdoc if term != ph and term in ph)
    return n > d

def check_phrase(phrase, term):
    return term in phrase

def gram_size(term):
    """
    Convenience func for getting n-gram length.
    """
    return len(term.split(' '))

def prune(tdocs):
	"""
	Prune terms which are totally subsumed by a phrase
	This could be better if it just removes the individual keywords
	that occur in a phrase for each time that phrase occurs.

... [truncated, 112 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S

import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
#import CalcDocSimilarity
import GensimCalcSimilarity
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs
import re
import numpy
import string
import sys  # import sys package, if not already imported
from textblob_de import TextBlobDE
from unidecode import unidecode

allowedRule			= [u'DET',u'ADJ',u'NOUN',u'VERB',u'ADV',u'ADJ']
####allowedRule				= [u'DET',u'ADJ',u'NOUN']
####allowedRule			= [u'ADJ',u'NOUN']

allowedRuleString		= "".join(allowedRule).replace("PUNCT","")


def createNERBasedSamples(text, markov, ndEntity):
	#markov += "In Berlin und Umgebung"
	goodNERFlagAny = any(x in markov for x in ndEntity)
	goodNERFlagAll = all(x in markov for x in ndEntity)
	'''
	print("\n -> Good Markov Result (NER-Based):")
	print(goodFlagAny)
	print(markov)
	print("\n\n")
	sys.exit(0)
	'''
	if goodNERFlagAll: 

... [truncated, 135 more lines] ...

from __future__ import division
import nltk
from nltk.corpus import wordnet as wn
from nltk.corpus import brown
import math
import numpy as np
import sys

# http://sujitpal.blogspot.de/2014/12/semantic-similarity-for-short-sentences.html
# Parameters to the algorithm. Currently set to values that was reported
# in the paper to produce "best" results.
ALPHA = 0.2
BETA = 0.45
ETA = 0.4
PHI = 0.2
DELTA = 0.85

brown_freqs = dict()
N = 0

######################### word similarity ##########################

def get_best_synset_pair(word_1, word_2):
    """ 
    Choose the pair with highest path similarity among all pairs. 
    Mimics pattern-seeking behavior of humans.
    """
    max_sim = -1.0
    synsets_1 = wn.synsets(word_1)
    synsets_2 = wn.synsets(word_2)
    if len(synsets_1) == 0 or len(synsets_2) == 0:
        return None, None
    else:
        max_sim = -1.0
        best_pair = None, None
        for synset_1 in synsets_1:
            for synset_2 in synsets_2:
               sim = wn.path_similarity(synset_1, synset_2)
               if sim > max_sim:
                   max_sim = sim
                   best_pair = synset_1, synset_2
        return best_pair

def length_dist(synset_1, synset_2):
    """
    Return a measure of the length of the shortest path in the semantic 
    ontology (Wordnet in our case as well as the paper's) between two 
    synsets.
    """
    l_dist = sys.maxint

... [truncated, 253 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
#import CalcDocSimilarity
import GensimCalcSimilarity
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs
import re

... [truncated, 206 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
import RuleBasedSamples
#import GensimCalcSimilarity
from RuleBasedSamples import *
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs

... [truncated, 73 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
import RuleBasedSamples
#import GensimCalcSimilarity
from RuleBasedSamples import *
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs

... [truncated, 73 more lines] ...