dowsery - 100biere - demo - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S

import logging
import gensim
from gensim import corpora
import codecs
from collections import defaultdict
from pprint import pprint  # pretty-printer
from gensim import corpora, models, similarities

#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

def calculateSimilarity(documents):
	
	stopword = "/home/100biere/demo/stopwordlist.de.txt"
	
	with codecs.open(stopword,'r', encoding='utf-8') as f:
		stopwords_tmp = f.read()
	f.close()
	stopwords = set(stopwords_tmp.strip().split())
	
	#texts = [[word for word in documents.lower().split() if word not in stopwords] for document in documents]
	texts = [word for word in documents.lower().split() if word not in stopwords]
	
	frequency = defaultdict(int)
	
	for text in texts:
		for token in text:
			frequency[token] += 1
	
	texts_final = [[token for token in text if frequency[token] > 1] for text in texts]
	dictionary = corpora.Dictionary(texts_final)
	dictionary.save('/home/100biere/demo/demo.dict')
	
	corpus = [dictionary.doc2bow(text) for text in texts_final]
	
	
	
	index = Similarity('/home/100biere/demo/demo.sim', corpus, num_features=12) # build the index
	similarities = index["Instagram"] # get similarities between the query and all index documents
	pprint(similarities)
	sys.exit(0)
	
	
	
	
	
	
	

... [truncated, 50 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
import time
import gensim
import nltk
import os, sys
import logging
import gensim
from gensim import corpora
import codecs
from collections import defaultdict
from pprint import pprint  # pretty-printer
from gensim import corpora, models, similarities

# https://radimrehurek.com/gensim/apiref.html

#pip install pyemd
from gensim.models import word2vec
from gensim.models import doc2vec

doc2vecpath		= "/home/100biere/demo/Doc2Vec.mod"
doc2vecpathDict	= "/home/100biere/demo/Doc2Vec.dict"
doc2vecpathCorp	= "/home/100biere/demo/Doc2Vec.corp"
stopword 		= "/home/100biere/demo/stopwordlist.de.txt"

def CalcDocSimilarity(documents, markov_sentence):
	
	if os.path.isfile(doc2vecpath):
		dictionary 	= corpora.Dictionary.load(doc2vecpathDict)
		#corpus = gensim.corpora.mmcorpus.MmCorpus.load(doc2vecpathCorp)
		#gensim.corpora.mmcorpus.MmCorpus.serialize(doc2vecpathCorp)
		corpus = gensim.corpora.mmcorpus.MmCorpus(doc2vecpathCorp)
		lsi = models.LsiModel.load(doc2vecpath)
		
		#lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=30, chunksize=100000)
		#index = similarities.MatrixSimilarity(lsi[corpus], num_features=len(corpus)) 
		vec_bow1 = dictionary.doc2bow(markov_sentence.lower().split())
		vec_lsi1 = lsi[vec_bow1] 
		#sims1 	= index[vec_lsi1] # perform a similarity query against the corpus
		#print("Markov:"),
		#print(vec_lsi1[0][1])
		if vec_lsi1[0][1]:
			return vec_lsi1[0][1]
		else:
			return 0.0
	else:
		# wenn documents als plain text string übergeben wird
		docs = [documents]
		
		with codecs.open(stopword,'r', encoding='utf-8') as f:

... [truncated, 135 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# https://github.com/BigFav/n-grams/blob/master/ngrams.py
from __future__ import division, unicode_literals
import argparse
import random
import re
import string
import sys
from copy import copy
from math import log10, isnan
from collections import Counter, OrderedDict, defaultdict, deque


"""
@description    n-grams, perplexity, and perplexity-based classification
@author         Favian Contreras <fnc4@cornell.edu>
"""


class Wrapper:
    """
    Wrapper created to pass by reference into a tuple, etc.
    While faster than using a list, I only did it as an exercise
    of overriding built-ins (as it only shows as faster when done
    over a million times).
    """
    def __init__(self):
        self.datum = None

    def __getitem__(self, index):
        return self.datum[index]

    def __iter__(self):
        return iter(self.datum)

    def __len__(self):
        return len(self.datum)

    def pop(self, index):
        return self.datum.pop(index)

    def set_datum(self, datum):
        self.datum = datum


class Ngrams:
    def __init__(self, opts):
        self.threshold = opts.threshold  # freq of words considered unknown
        self.start_token = "<s>"

... [truncated, 925 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
"""
A naive keyword extractor which just pulls out nouns and noun phrases.

Was using the PerceptronTagger is _way_ faster than NLTK's default tagger, and more accurate to boot.
See <http://stevenloria.com/tutorial-state-of-the-art-part-of-speech-tagging-in-textblob/>.

However, it complicates the library's installation, and the spacy tagger is quite fast and good too.

https://github.com/frnsys/broca/blob/master/broca/common/util.py

"""
from textblob_de import TextBlobDE
#import spacy
import codecs

CFG = {
    ('NNP', 'NNP'): 'NNP',
    ('NN', 'NN'): 'NNI',
    ('NNI', 'NN'): 'NNI',
    ('JJ', 'JJ'): 'JJ',
    ('JJ', 'NN'): 'NNI',
}

def check_term(tdoc, term):
    if term not in tdoc:
        return False

    # If this term occurs outside of a phrase,
    # it is no longer a candidate
    n = tdoc.count(term)
    # Count phrases that contain t
    d = sum(1 for ph in tdoc if term != ph and term in ph)
    return n > d

def check_phrase(phrase, term):
    return term in phrase

def gram_size(term):
    """
    Convenience func for getting n-gram length.
    """
    return len(term.split(' '))

def prune(tdocs):
	"""
	Prune terms which are totally subsumed by a phrase
	This could be better if it just removes the individual keywords
	that occur in a phrase for each time that phrase occurs.

... [truncated, 112 more lines] ...

"""
A naive keyword extractor which just pulls out nouns and noun phrases.

Was using the PerceptronTagger is _way_ faster than NLTK's default tagger, and more accurate to boot.
See <http://stevenloria.com/tutorial-state-of-the-art-part-of-speech-tagging-in-textblob/>.

However, it complicates the library's installation, and the spacy tagger is quite fast and good too.
"""

import spacy

CFG = {
    ('NNP', 'NNP'): 'NNP',
    ('NN', 'NN'): 'NNI',
    ('NNI', 'NN'): 'NNI',
    ('JJ', 'JJ'): 'JJ',
    ('JJ', 'NN'): 'NNI',
}


class POSTokenizer(Tokenizer):
    
	def prune(tdocs):
		"""
		Prune terms which are totally subsumed by a phrase
		This could be better if it just removes the individual keywords
		that occur in a phrase for each time that phrase occurs.
		"""
		all_terms = set([t for toks in tdocs for t in toks])
		terms = set()
		phrases = set()
		for t in all_terms:
			if gram_size(t) > 1:
				phrases.add(t)
			else:
				terms.add(t)

		# Identify candidates for redundant terms (1-gram terms found in a phrase)
		redundant = set()
		for t in terms:
			if any(t in ph for ph in phrases):
				redundant.add(t)

		# Search all documents to check that these terms occur
		# only in a phrase. If not, remove it as a candidate.
		# This could be more efficient
		cleared = set()
		for t in redundant:
			if any(check_term(d, term=t) for d in tdocs):
				cleared.add(t)

... [truncated, 68 more lines] ...

from __future__ import division
import nltk
from nltk.corpus import wordnet as wn
from nltk.corpus import brown
import math
import numpy as np
import sys

# http://sujitpal.blogspot.de/2014/12/semantic-similarity-for-short-sentences.html
# Parameters to the algorithm. Currently set to values that was reported
# in the paper to produce "best" results.
ALPHA = 0.2
BETA = 0.45
ETA = 0.4
PHI = 0.2
DELTA = 0.85

brown_freqs = dict()
N = 0

######################### word similarity ##########################

def get_best_synset_pair(word_1, word_2):
    """ 
    Choose the pair with highest path similarity among all pairs. 
    Mimics pattern-seeking behavior of humans.
    """
    max_sim = -1.0
    synsets_1 = wn.synsets(word_1)
    synsets_2 = wn.synsets(word_2)
    if len(synsets_1) == 0 or len(synsets_2) == 0:
        return None, None
    else:
        max_sim = -1.0
        best_pair = None, None
        for synset_1 in synsets_1:
            for synset_2 in synsets_2:
               sim = wn.path_similarity(synset_1, synset_2)
               if sim > max_sim:
                   max_sim = sim
                   best_pair = synset_1, synset_2
        return best_pair

def length_dist(synset_1, synset_2):
    """
    Return a measure of the length of the shortest path in the semantic 
    ontology (Wordnet in our case as well as the paper's) between two 
    synsets.
    """
    l_dist = sys.maxint

... [truncated, 253 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S

import logging
from gensim import corpora
import codecs
from collections import defaultdict
from pprint import pprint  # pretty-printer

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)


def calculateSimilarity(document):
	stopword = "/home/100biere/demo/stopwordlist.de.txt"
	
	with codecs.open(stopword,'r', encoding='utf-8') as f:
		stopwords_tmp = f.read()
	f.close()
	stopwords = set(stopwords_tmp.split("\n"))
	
	texts = [[word for word in document.lower().split() if word not in stoplist] for document in documents]
	
	frequency = defaultdict(int)
	
	for text in texts:
		for token in text:
			frequency[token] += 1
	
	texts = [[token for token in text if frequency[token] > 1] for text in texts]
	pprint(texts)
	





def get_similarity_scores_all_pairs(texts):
    """Takes a list of strings as input and returns a matrix of cosine similarity values where element [m][n] represents the similarity between text m and text n.
    >>> get_similarity_scores_all_pairs(['apple banana', 'banana cherry'])
    array([[ 1.,  0.],
           [ 0.,  1.]])
    """
    n = len(texts)
    all_similarities = numpy.empty(shape=(n, n))
    similarity_index = __get_tfidf_similarity_index(texts)
  
    for i in range(n):
        all_similarities[i] = similarity_index.similarity_by_id(i)
    return all_similarities
  

... [truncated, 23 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import polyglot
from polyglot.text import Text, Word
import NP
#import language_check
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
from spacy.de import German
import base64
import os.path
import json
import pprint
import codecs
import nltk

... [truncated, 141 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
import polyglot
from polyglot.text import Text, Word
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
import os.path
import pprint

... [truncated, 190 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
import polyglot
from polyglot.text import Text, Word
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
import os.path
import pprint

... [truncated, 254 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
#import CalcDocSimilarity
import GensimCalcSimilarity
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs
import re

... [truncated, 215 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
#import CalcDocSimilarity
import GensimCalcSimilarity
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs
import re

... [truncated, 206 more lines] ...

"""Learn to estimate functions from examples. (Chapters 18-20)"""

from utils import *
import copy, heapq, math, random
from collections import defaultdict

#______________________________________________________________________________

def rms_error(predictions, targets):
    return math.sqrt(ms_error(predictions, targets))

def ms_error(predictions, targets):
    return mean([(p - t)**2 for p, t in zip(predictions, targets)])

def mean_error(predictions, targets):
    return mean([abs(p - t) for p, t in zip(predictions, targets)])

def mean_boolean_error(predictions, targets):
    return mean([(p != t)   for p, t in zip(predictions, targets)])

#______________________________________________________________________________

class DataSet:
    """A data set for a machine learning problem.  It has the following fields:

    d.examples    A list of examples.  Each one is a list of attribute values.
    d.attrs       A list of integers to index into an example, so example[attr]
                  gives a value. Normally the same as range(len(d.examples[0])).
    d.attrnames   Optional list of mnemonic names for corresponding attrs.
    d.target      The attribute that a learning algorithm will try to predict.
                  By default the final attribute.
    d.inputs      The list of attrs without the target.
    d.values      A list of lists: each sublist is the set of possible
                  values for the corresponding attribute. If initially None,
                  it is computed from the known examples by self.setproblem.
                  If not None, an erroneous value raises ValueError.
    d.distance    A function from a pair of examples to a nonnegative number.
                  Should be symmetric, etc. Defaults to mean_boolean_error
                  since that can handle any field types.
    d.name        Name of the data set (for output display only).
    d.source      URL or other source where the data came from.

    Normally, you call the constructor and you're done; then you just
    access fields like d.examples and d.target and d.inputs."""

    def __init__(self, examples=None, attrs=None, attrnames=None, target=-1,
                 inputs=None, values=None, distance=mean_boolean_error,
                 name='', source='', exclude=()):
        """Accepts any of DataSet's fields.  Examples can also be a
        string or file from which to parse examples using parse_csv.

... [truncated, 544 more lines] ...

# coding=UTF-8
import nltk
from nltk.corpus import brown

# This is a fast and simple noun phrase extractor (based on NLTK)
# Feel free to use it, just keep a link back to this post
# http://thetokenizer.com/2013/05/09/efficient-way-to-extract-the-main-topics-of-a-sentence/
# Create by Shlomi Babluki
# May, 2013


# This is our fast Part of Speech tagger
#############################################################################
brown_train = brown.tagged_sents(categories='news')
regexp_tagger = nltk.RegexpTagger(
    [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'),
     (r'(-|:|;)$', ':'),
     (r'\'*$', 'MD'),
     (r'(The|the|A|a|An|an)$', 'AT'),
     (r'.*able$', 'JJ'),
     (r'^[A-Z].*$', 'NNP'),
     (r'.*ness$', 'NN'),
     (r'.*ly$', 'RB'),
     (r'.*s$', 'NNS'),
     (r'.*ing$', 'VBG'),
     (r'.*ed$', 'VBD'),
     (r'.*', 'NN')
])
unigram_tagger = nltk.UnigramTagger(brown_train, backoff=regexp_tagger)
bigram_tagger = nltk.BigramTagger(brown_train, backoff=unigram_tagger)
#############################################################################


# This is our semi-CFG; Extend it according to your own needs
#############################################################################
cfg = {}
cfg["NNP+NNP"] = "NNP"
cfg["NN+NN"] = "NNI"
cfg["NNI+NN"] = "NNI"
cfg["JJ+JJ"] = "JJ"
cfg["JJ+NN"] = "NNI"
#############################################################################


class NPExtractor(object):

    def __init__(self, sentence):
        self.sentence = sentence

    # Split the sentence into singlw words/tokens

... [truncated, 62 more lines] ...

# coding=UTF-8
import nltk
from nltk.corpus import brown

# This is a fast and simple noun phrase extractor (based on NLTK)
# Feel free to use it, just keep a link back to this post
# http://thetokenizer.com/2013/05/09/efficient-way-to-extract-the-main-topics-of-a-sentence/
# Create by Shlomi Babluki
# May, 2013


# This is our fast Part of Speech tagger
#############################################################################
brown_train = brown.tagged_sents(categories='news')
regexp_tagger = nltk.RegexpTagger(
    [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'),
     (r'(-|:|;)$', ':'),
     (r'\'*$', 'MD'),
     (r'(The|the|A|a|An|an)$', 'AT'),
     (r'.*able$', 'JJ'),
     (r'^[A-Z].*$', 'NNP'),
     (r'.*ness$', 'NN'),
     (r'.*ly$', 'RB'),
     (r'.*s$', 'NNS'),
     (r'.*ing$', 'VBG'),
     (r'.*ed$', 'VBD'),
     (r'.*', 'NN')
])
unigram_tagger = nltk.UnigramTagger(brown_train, backoff=regexp_tagger)
bigram_tagger = nltk.BigramTagger(brown_train, backoff=unigram_tagger)
#############################################################################


# This is our semi-CFG; Extend it according to your own needs
#############################################################################
cfg = {}
cfg["NNP+NNP"] = "NNP"
cfg["NN+NN"] = "NNI"
cfg["NNI+NN"] = "NNI"
cfg["JJ+JJ"] = "JJ"
cfg["JJ+NN"] = "NNI"
#############################################################################


class NPExtractor(object):

    def __init__(self, sentence):
        self.sentence = sentence

    # Split the sentence into singlw words/tokens

... [truncated, 62 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import polyglot
from polyglot.text import Text, Word
import NP
#import language_check
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
from spacy.de import German
import base64
import os.path
import json
import pprint
import codecs
import nltk

... [truncated, 38 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')
import sys  # import sys package, if not already imported
#from textblob_de import TextBlobDE
import spacy

t1 = u"ich gehe spazieren und esse ein Eis."
t2 = u"das warme Essen schmeckt sehr lecker, denn es ist sehr schmackhaft zubereitet."

#nlp 		= spacy.load('de', tagger=True, parser=True, entity=True)
#nlp 		= spacy.load('de')
nlp 		= spacy.de.German()
toks1 		= nlp(t1)
toks2 		= nlp(t2)
print(toks1), 
print("\n")
for token1 in toks1:
	print(token1, token1.pos, token1.pos_)

print(toks2),
print("\n")
for token2 in toks2:

... [truncated, 8 more lines] ...

# Implementation of RAKE - Rapid Automtic Keyword Exraction algorithm
# as described in:
# Rose, S., D. Engel, N. Cramer, and W. Cowley (2010). 
# Automatic keyword extraction from indi-vidual documents. 
# In M. W. Berry and J. Kogan (Eds.), Text Mining: Applications and Theory.unknown: John Wiley and Sons, Ltd.
#
# NOTE: The original code (from https://github.com/aneesha/RAKE)
# has been extended by a_medelyan (zelandiya)
# with a set of heuristics to decide whether a phrase is an acceptable candidate
# as well as the ability to set frequency and phrase length parameters
# important when dealing with longer documents

from __future__ import absolute_import
from __future__ import print_function
import re
import operator
import six
import nltk
from six.moves import range

debug = False
test = False

def is_number(s):
    try:
        float(s) if '.' in s else int(s)
        return True
    except ValueError:
        return False


def load_stop_words(stop_word_file):
    """
    Utility function to load stop words from a file and return as a list of words
    @param stop_word_file Path and file name of a file containing stop words.
    @return list A list of stop words.
    """
    stop_words = []
    for line in open(stop_word_file):
        if line.strip()[0:1] != "#":
            for word in line.split():  # in case more than one per line
                stop_words.append(word)
    return stop_words


def separate_words(text, min_word_return_size):
    """
    Utility function to return a list of all words that are have a length greater than a specified number of characters.
    @param text The text that must be split in to words.
    @param min_word_return_size The minimum no of characters a word must have to be included.

... [truncated, 167 more lines] ...

"""Search (Chapters 3-4)

The way to use this code is to subclass Problem to create a class of problems,
then create problem instances and solve them with calls to the various search
functions."""

from utils import *
import math, random, sys, time, bisect, string

#______________________________________________________________________________

class Problem(object):
    """The abstract class for a formal problem.  You should subclass
    this and implement the methods actions and result, and possibly
    __init__, goal_test, and path_cost. Then you will create instances
    of your subclass and solve them with the various search functions."""

    def __init__(self, initial, goal=None):
        """The constructor specifies the initial state, and possibly a goal
        state, if there is a unique goal.  Your subclass's constructor can add
        other arguments."""
        self.initial = initial; self.goal = goal

    def actions(self, state):
        """Return the actions that can be executed in the given
        state. The result would typically be a list, but if there are
        many actions, consider yielding them one at a time in an
        iterator, rather than building them all at once."""
        abstract

    def result(self, state, action):
        """Return the state that results from executing the given
        action in the given state. The action must be one of
        self.actions(state)."""
        abstract

    def goal_test(self, state):
        """Return True if the state is a goal. The default method compares the
        state to self.goal, as specified in the constructor. Override this
        method if checking against a single self.goal is not enough."""
        return state == self.goal

    def path_cost(self, c, state1, action, state2):
        """Return the cost of a solution path that arrives at state2 from
        state1 via action, assuming cost c to get up to state1. If the problem
        is such that the path doesn't matter, this function will only look at
        state2.  If the path does matter, it will consider c and maybe state1
        and action. The default method costs 1 for every step in the path."""
        return c + 1


... [truncated, 785 more lines] ...

"""Provide some widely useful utilities. Safe for "from utils import *".

"""

from __future__ import generators
import operator, math, random, copy, sys, os.path, bisect, re

assert (2,5) <= sys.version_info < (3,), """\
This code is meant for Python 2.5 through 2.7.
You might find that the parts you care about still work in older
Pythons or happen to work in newer ones, but you're on your own --
edit utils.py if you want to try it."""

#______________________________________________________________________________
# Compatibility with Python 2.2, 2.3, and 2.4

# The AIMA code was originally designed to run in Python 2.2 and up.
# The first part of this file implements for Python 2.2 through 2.4
# the parts of 2.5 that the original code relied on. Now we're
# starting to go beyond what can be filled in this way, but here's
# the compatibility code still since it doesn't hurt:

try: bool, True, False ## Introduced in 2.3
except NameError:
    class bool(int):
        "Simple implementation of Booleans, as in PEP 285"
        def __init__(self, val): self.val = val
        def __int__(self): return self.val
        def __repr__(self): return ('False', 'True')[self.val]

    True, False = bool(1), bool(0)

try: sum ## Introduced in 2.3
except NameError:
    def sum(seq, start=0):
        """Sum the elements of seq.
        >>> sum([1, 2, 3])
        6
        """
        return reduce(operator.add, seq, start)

try: enumerate  ## Introduced in 2.3
except NameError:
    def enumerate(collection):
        """Return an iterator that enumerates pairs of (i, c[i]). PEP 279.
        >>> list(enumerate('abc'))
        [(0, 'a'), (1, 'b'), (2, 'c')]
        """
        ## Copied from PEP 279
        i = 0

... [truncated, 917 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

start_time = time.time()

import language_check
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
from spacy.de import German
import base64
import os.path
import json
import pprint
import codecs
import nltk
import re
from subprocess import call
import sys  # import sys package, if not already imported
from textblob_de import TextBlobDE as TextBlob

def only_letters(string):
	#s = re.sub('[^0-9a-zA-Z]+', '*', s)
	lol = re.findall(r'[^A-Za-z-\s\d]',string)
	#pprint.pprint("Suche: "+string),
	#pprint.pprint(lol)
	if len(lol) > 0:
		return 0
	else:
		#pprint.pprint(lol)
		#pprint.pprint(len(lol))
		return 1
		
	return 0


#de_nlp 		= spacy.load('de', tagger=True, parser=True, entity=True)
de_nlp 		= spacy.de.German()

... [truncated, 175 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

start_time = time.time()

# http://polyglot.readthedocs.io/en/latest/Installation.html
import polyglot
from polyglot.text import Text, Word
#import rake
import NP
#import language_check
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
from spacy.de import German
import base64
import os.path
import json
import pprint
import codecs
import nltk
import re
from subprocess import call
import sys  # import sys package, if not already imported
from textblob_de import TextBlobDE
from unidecode import unidecode

def remove_non_ascii(text):
    return unidecode(unicode(text, encoding = "utf-8"))
	
def only_letters(string):
	#s = re.sub('[^0-9a-zA-Z]+', '*', s)
	lol = re.findall(r'[^A-Za-z-\s\d]',string)
	#pprint.pprint("Suche: "+string),
	#pprint.pprint(lol)
	if len(lol) > 0:
		return 0
	else:

... [truncated, 236 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

start_time = time.time()

# http://polyglot.readthedocs.io/en/latest/Installation.html
import polyglot
from polyglot.text import Text, Word
import rake
import NP
import itertools
#import language_check
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
from spacy.de import German
import base64
import os.path
import json
import pprint
import codecs
import nltk
import re
import string
from subprocess import call
import sys  # import sys package, if not already imported
from textblob_de import TextBlobDE

def only_letters(string):
	#s = re.sub('[^0-9a-zA-Z]+', '*', s)
	lol = re.findall(r'[^A-Za-z-\s\d]',string)
	#pprint.pprint("Suche: "+string),
	#pprint.pprint(lol)
	if len(lol) > 0:
		return 0
	else:
		#pprint.pprint(lol)
		#pprint.pprint(len(lol))

... [truncated, 211 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import polyglot
from polyglot.text import Text, Word
import rake
import NP
#import language_check
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
from spacy.de import German
import base64
import os.path
import json
import pprint
import codecs
import nltk
import re
import string
from subprocess import call
import sys  # import sys package, if not already imported
from textblob_de import TextBlobDE

def only_letters(string):
	#s = re.sub('[^0-9a-zA-Z]+', '*', s)
	lol = re.findall(r'[^A-Za-z-\s\d]',string)
	#pprint.pprint("Suche: "+string),
	#pprint.pprint(lol)
	if len(lol) > 0:
		return 0
	else:
		#pprint.pprint(lol)

... [truncated, 211 more lines] ...