dowsery - 100biere - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S

import logging
import gensim
from gensim import corpora
import codecs
from collections import defaultdict
from pprint import pprint  # pretty-printer
from gensim import corpora, models, similarities

#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

def calculateSimilarity(documents):
	
	stopword = "/home/100biere/demo/stopwordlist.de.txt"
	
	with codecs.open(stopword,'r', encoding='utf-8') as f:
		stopwords_tmp = f.read()
	f.close()
	stopwords = set(stopwords_tmp.strip().split())
	
	#texts = [[word for word in documents.lower().split() if word not in stopwords] for document in documents]
	texts = [word for word in documents.lower().split() if word not in stopwords]
	
	frequency = defaultdict(int)
	
	for text in texts:
		for token in text:
			frequency[token] += 1
	
	texts_final = [[token for token in text if frequency[token] > 1] for text in texts]
	dictionary = corpora.Dictionary(texts_final)
	dictionary.save('/home/100biere/demo/demo.dict')
	
	corpus = [dictionary.doc2bow(text) for text in texts_final]
	
	
	
	index = Similarity('/home/100biere/demo/demo.sim', corpus, num_features=12) # build the index
	similarities = index["Instagram"] # get similarities between the query and all index documents
	pprint(similarities)
	sys.exit(0)
	
	
	
	
	
	
	

... [truncated, 50 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
import time
import gensim
import nltk
import os, sys
import logging
import gensim
from gensim import corpora
import codecs
from collections import defaultdict
from pprint import pprint  # pretty-printer
from gensim import corpora, models, similarities

# https://radimrehurek.com/gensim/apiref.html

#pip install pyemd
from gensim.models import word2vec
from gensim.models import doc2vec

doc2vecpath		= "/home/100biere/demo/Doc2Vec.mod"
doc2vecpathDict	= "/home/100biere/demo/Doc2Vec.dict"
doc2vecpathCorp	= "/home/100biere/demo/Doc2Vec.corp"
stopword 		= "/home/100biere/demo/stopwordlist.de.txt"

def CalcDocSimilarity(documents, markov_sentence):
	
	if os.path.isfile(doc2vecpath):
		dictionary 	= corpora.Dictionary.load(doc2vecpathDict)
		#corpus = gensim.corpora.mmcorpus.MmCorpus.load(doc2vecpathCorp)
		#gensim.corpora.mmcorpus.MmCorpus.serialize(doc2vecpathCorp)
		corpus = gensim.corpora.mmcorpus.MmCorpus(doc2vecpathCorp)
		lsi = models.LsiModel.load(doc2vecpath)
		
		#lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=30, chunksize=100000)
		#index = similarities.MatrixSimilarity(lsi[corpus], num_features=len(corpus)) 
		vec_bow1 = dictionary.doc2bow(markov_sentence.lower().split())
		vec_lsi1 = lsi[vec_bow1] 
		#sims1 	= index[vec_lsi1] # perform a similarity query against the corpus
		#print("Markov:"),
		#print(vec_lsi1[0][1])
		if vec_lsi1[0][1]:
			return vec_lsi1[0][1]
		else:
			return 0.0
	else:
		# wenn documents als plain text string übergeben wird
		docs = [documents]
		
		with codecs.open(stopword,'r', encoding='utf-8') as f:

... [truncated, 135 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# https://github.com/BigFav/n-grams/blob/master/ngrams.py
from __future__ import division, unicode_literals
import argparse
import random
import re
import string
import sys
from copy import copy
from math import log10, isnan
from collections import Counter, OrderedDict, defaultdict, deque


"""
@description    n-grams, perplexity, and perplexity-based classification
@author         Favian Contreras <fnc4@cornell.edu>
"""


class Wrapper:
    """
    Wrapper created to pass by reference into a tuple, etc.
    While faster than using a list, I only did it as an exercise
    of overriding built-ins (as it only shows as faster when done
    over a million times).
    """
    def __init__(self):
        self.datum = None

    def __getitem__(self, index):
        return self.datum[index]

    def __iter__(self):
        return iter(self.datum)

    def __len__(self):
        return len(self.datum)

    def pop(self, index):
        return self.datum.pop(index)

    def set_datum(self, datum):
        self.datum = datum


class Ngrams:
    def __init__(self, opts):
        self.threshold = opts.threshold  # freq of words considered unknown
        self.start_token = "<s>"

... [truncated, 925 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
"""
A naive keyword extractor which just pulls out nouns and noun phrases.

Was using the PerceptronTagger is _way_ faster than NLTK's default tagger, and more accurate to boot.
See <http://stevenloria.com/tutorial-state-of-the-art-part-of-speech-tagging-in-textblob/>.

However, it complicates the library's installation, and the spacy tagger is quite fast and good too.

https://github.com/frnsys/broca/blob/master/broca/common/util.py

"""
from textblob_de import TextBlobDE
#import spacy
import codecs

CFG = {
    ('NNP', 'NNP'): 'NNP',
    ('NN', 'NN'): 'NNI',
    ('NNI', 'NN'): 'NNI',
    ('JJ', 'JJ'): 'JJ',
    ('JJ', 'NN'): 'NNI',
}

def check_term(tdoc, term):
    if term not in tdoc:
        return False

    # If this term occurs outside of a phrase,
    # it is no longer a candidate
    n = tdoc.count(term)
    # Count phrases that contain t
    d = sum(1 for ph in tdoc if term != ph and term in ph)
    return n > d

def check_phrase(phrase, term):
    return term in phrase

def gram_size(term):
    """
    Convenience func for getting n-gram length.
    """
    return len(term.split(' '))

def prune(tdocs):
	"""
	Prune terms which are totally subsumed by a phrase
	This could be better if it just removes the individual keywords
	that occur in a phrase for each time that phrase occurs.

... [truncated, 112 more lines] ...

"""
A naive keyword extractor which just pulls out nouns and noun phrases.

Was using the PerceptronTagger is _way_ faster than NLTK's default tagger, and more accurate to boot.
See <http://stevenloria.com/tutorial-state-of-the-art-part-of-speech-tagging-in-textblob/>.

However, it complicates the library's installation, and the spacy tagger is quite fast and good too.
"""

import spacy

CFG = {
    ('NNP', 'NNP'): 'NNP',
    ('NN', 'NN'): 'NNI',
    ('NNI', 'NN'): 'NNI',
    ('JJ', 'JJ'): 'JJ',
    ('JJ', 'NN'): 'NNI',
}


class POSTokenizer(Tokenizer):
    
	def prune(tdocs):
		"""
		Prune terms which are totally subsumed by a phrase
		This could be better if it just removes the individual keywords
		that occur in a phrase for each time that phrase occurs.
		"""
		all_terms = set([t for toks in tdocs for t in toks])
		terms = set()
		phrases = set()
		for t in all_terms:
			if gram_size(t) > 1:
				phrases.add(t)
			else:
				terms.add(t)

		# Identify candidates for redundant terms (1-gram terms found in a phrase)
		redundant = set()
		for t in terms:
			if any(t in ph for ph in phrases):
				redundant.add(t)

		# Search all documents to check that these terms occur
		# only in a phrase. If not, remove it as a candidate.
		# This could be more efficient
		cleared = set()
		for t in redundant:
			if any(check_term(d, term=t) for d in tdocs):
				cleared.add(t)

... [truncated, 68 more lines] ...

from __future__ import division
import nltk
from nltk.corpus import wordnet as wn
from nltk.corpus import brown
import math
import numpy as np
import sys

# http://sujitpal.blogspot.de/2014/12/semantic-similarity-for-short-sentences.html
# Parameters to the algorithm. Currently set to values that was reported
# in the paper to produce "best" results.
ALPHA = 0.2
BETA = 0.45
ETA = 0.4
PHI = 0.2
DELTA = 0.85

brown_freqs = dict()
N = 0

######################### word similarity ##########################

def get_best_synset_pair(word_1, word_2):
    """ 
    Choose the pair with highest path similarity among all pairs. 
    Mimics pattern-seeking behavior of humans.
    """
    max_sim = -1.0
    synsets_1 = wn.synsets(word_1)
    synsets_2 = wn.synsets(word_2)
    if len(synsets_1) == 0 or len(synsets_2) == 0:
        return None, None
    else:
        max_sim = -1.0
        best_pair = None, None
        for synset_1 in synsets_1:
            for synset_2 in synsets_2:
               sim = wn.path_similarity(synset_1, synset_2)
               if sim > max_sim:
                   max_sim = sim
                   best_pair = synset_1, synset_2
        return best_pair

def length_dist(synset_1, synset_2):
    """
    Return a measure of the length of the shortest path in the semantic 
    ontology (Wordnet in our case as well as the paper's) between two 
    synsets.
    """
    l_dist = sys.maxint

... [truncated, 253 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S

import logging
from gensim import corpora
import codecs
from collections import defaultdict
from pprint import pprint  # pretty-printer

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)


def calculateSimilarity(document):
	stopword = "/home/100biere/demo/stopwordlist.de.txt"
	
	with codecs.open(stopword,'r', encoding='utf-8') as f:
		stopwords_tmp = f.read()
	f.close()
	stopwords = set(stopwords_tmp.split("\n"))
	
	texts = [[word for word in document.lower().split() if word not in stoplist] for document in documents]
	
	frequency = defaultdict(int)
	
	for text in texts:
		for token in text:
			frequency[token] += 1
	
	texts = [[token for token in text if frequency[token] > 1] for text in texts]
	pprint(texts)
	





def get_similarity_scores_all_pairs(texts):
    """Takes a list of strings as input and returns a matrix of cosine similarity values where element [m][n] represents the similarity between text m and text n.
    >>> get_similarity_scores_all_pairs(['apple banana', 'banana cherry'])
    array([[ 1.,  0.],
           [ 0.,  1.]])
    """
    n = len(texts)
    all_similarities = numpy.empty(shape=(n, n))
    similarity_index = __get_tfidf_similarity_index(texts)
  
    for i in range(n):
        all_similarities[i] = similarity_index.similarity_by_id(i)
    return all_similarities
  

... [truncated, 23 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import polyglot
from polyglot.text import Text, Word
import NP
#import language_check
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
from spacy.de import German
import base64
import os.path
import json
import pprint
import codecs
import nltk

... [truncated, 141 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
import polyglot
from polyglot.text import Text, Word
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
import os.path
import pprint

... [truncated, 190 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
import polyglot
from polyglot.text import Text, Word
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import gensim, logging
import markovify
from langdetect import detect
import spacy
import os.path
import pprint

... [truncated, 254 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
#import CalcDocSimilarity
import GensimCalcSimilarity
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs
import re

... [truncated, 215 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S
# python -m spacy.en.download python -m spacy.de.download
# https://spacy.io/docs/#tutorials
#https://www.w3.org/services/html2txt
# CSS: http://codepen.io/explosion/pen/xEpgKz
# CSS 2: https://explosion.ai/blog/displacy-ent-named-entity-visualizer
import time

#pip install --upgrade 3to2
#pip install --upgrade language-check

### grammar checker
######## https://github.com/lpenz/atdtool/blob/master/atdtool/__init__.py
###########http://stackoverflow.com/questions/10252448/how-to-check-whether-a-sentence-is-correct-simple-grammar-check-in-python --> https://pypi.python.org/pypi/grammar-check https://pypi.python.org/pypi/language-check

#Statistical spell- and (occasional) grammar-checker. http://lindat.mff.cuni.cz/services/korektor
#https://github.com/ufal/korektor

	#/home/100biere/demo/tensorflow/models/syntaxnet#
# https://wiki.python.org/moin/LanguageParsing
# http://nlp.stanford.edu/software/lex-parser.shtml

#https://github.com/jsvine/markovify

# spell correction: http://norvig.com/spell-correct.html

# grammar correct: http://www.abisource.com/projects/link-grammar/#download

# python based grammar check based on learning https://www.openhub.net/p/grac

# http://www.decontextualize.com/teaching/rwet/n-grams-and-markov-chains/

start_time = time.time()
import os
os.system('clear')

# http://polyglot.readthedocs.io/en/latest/Installation.html
import NP
#import CalcDocSimilarity
import GensimCalcSimilarity
import polyglot
from polyglot.text import Text, Word
import markovify
from langdetect import detect
import spacy
import os.path
import pprint
import codecs
import re

... [truncated, 206 more lines] ...

"""Learn to estimate functions from examples. (Chapters 18-20)"""

from utils import *
import copy, heapq, math, random
from collections import defaultdict

#______________________________________________________________________________

def rms_error(predictions, targets):
    return math.sqrt(ms_error(predictions, targets))

def ms_error(predictions, targets):
    return mean([(p - t)**2 for p, t in zip(predictions, targets)])

def mean_error(predictions, targets):
    return mean([abs(p - t) for p, t in zip(predictions, targets)])

def mean_boolean_error(predictions, targets):
    return mean([(p != t)   for p, t in zip(predictions, targets)])

#______________________________________________________________________________

class DataSet:
    """A data set for a machine learning problem.  It has the following fields:

    d.examples    A list of examples.  Each one is a list of attribute values.
    d.attrs       A list of integers to index into an example, so example[attr]
                  gives a value. Normally the same as range(len(d.examples[0])).
    d.attrnames   Optional list of mnemonic names for corresponding attrs.
    d.target      The attribute that a learning algorithm will try to predict.
                  By default the final attribute.
    d.inputs      The list of attrs without the target.
    d.values      A list of lists: each sublist is the set of possible
                  values for the corresponding attribute. If initially None,
                  it is computed from the known examples by self.setproblem.
                  If not None, an erroneous value raises ValueError.
    d.distance    A function from a pair of examples to a nonnegative number.
                  Should be symmetric, etc. Defaults to mean_boolean_error
                  since that can handle any field types.
    d.name        Name of the data set (for output display only).
    d.source      URL or other source where the data came from.

    Normally, you call the constructor and you're done; then you just
    access fields like d.examples and d.target and d.inputs."""

    def __init__(self, examples=None, attrs=None, attrnames=None, target=-1,
                 inputs=None, values=None, distance=mean_boolean_error,
                 name='', source='', exclude=()):
        """Accepts any of DataSet's fields.  Examples can also be a
        string or file from which to parse examples using parse_csv.

... [truncated, 544 more lines] ...

# coding=UTF-8
import nltk
from nltk.corpus import brown

# This is a fast and simple noun phrase extractor (based on NLTK)
# Feel free to use it, just keep a link back to this post
# http://thetokenizer.com/2013/05/09/efficient-way-to-extract-the-main-topics-of-a-sentence/
# Create by Shlomi Babluki
# May, 2013


# This is our fast Part of Speech tagger
#############################################################################
brown_train = brown.tagged_sents(categories='news')
regexp_tagger = nltk.RegexpTagger(
    [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'),
     (r'(-|:|;)$', ':'),
     (r'\'*$', 'MD'),
     (r'(The|the|A|a|An|an)$', 'AT'),
     (r'.*able$', 'JJ'),
     (r'^[A-Z].*$', 'NNP'),
     (r'.*ness$', 'NN'),
     (r'.*ly$', 'RB'),
     (r'.*s$', 'NNS'),
     (r'.*ing$', 'VBG'),
     (r'.*ed$', 'VBD'),
     (r'.*', 'NN')
])
unigram_tagger = nltk.UnigramTagger(brown_train, backoff=regexp_tagger)
bigram_tagger = nltk.BigramTagger(brown_train, backoff=unigram_tagger)
#############################################################################


# This is our semi-CFG; Extend it according to your own needs
#############################################################################
cfg = {}
cfg["NNP+NNP"] = "NNP"
cfg["NN+NN"] = "NNI"
cfg["NNI+NN"] = "NNI"
cfg["JJ+JJ"] = "JJ"
cfg["JJ+NN"] = "NNI"
#############################################################################


class NPExtractor(object):

    def __init__(self, sentence):
        self.sentence = sentence

    # Split the sentence into singlw words/tokens

... [truncated, 62 more lines] ...