dowsery - dowsery-demo - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/env python
# https://developers.google.com/custom-search/docs/xml_results#countryCodes
#https://www.linkedin.com/countserv/count/share?format=jsonp&url=https://www.buzzerstar.com 
# pip install --upgrade spacy tensorflow gensim sumy keras markovify google-api-python-client beautifulsoup4
#from sphinxapi import *
import sys, time
import MySQLdb as mdb

import hashlib
import pickle
#import argparse
import json
#import sys

import googleapiclient.discovery

import re
import numpy as np
import gensim
import nltk
import os, sys, logging, re
from gensim import corpora, models, similarities
from gensim.models import word2vec
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
from gensim.models import word2vec
from gensim.models import doc2vec
from gensim.models.ldamulticore import LdaMulticore
#import spacy# pip3 install --upgrade spacy
#from spacy.en import English
#from spacy.de import German
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from nltk.tokenize import PunktSentenceTokenizer
from nltk.stem.snowball import SnowballStemmer

#pip3 install stop-words
from stop_words import get_stop_words
stopwordsDE = get_stop_words('de')

from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.sparse import csr_matrix
cachedStopWords = stopwords.words("german")

# load nltk's English stopwords as variable called 'stopwords'

... [truncated, 948 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/env python
"""
python3 DowserySummaryPreprepareADV.py

python3 textsum_data_convert.py --command text_to_binary --in_directories train_data --out_files binary_data2/dowsery-train.bin,binary_data2/dowsery-validation.bin,binary_data2/dowsery-test.bin --split 1.0,0,0

python3 textsum_data_convert.py --command text_to_vocabulary --in_directories train_data --out_files binary_data2/vocab.bin

python3 pointer-generator/run_summarization.py --mode=train --data_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data2/dowsery-* --vocab_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data2/vocab.bin --log_root=/home/Framework/Prototyp/tensorflow-text-summary/binary_data2 --exp_name=myexperiment2

-> Single Pass
python3 pointer-generator/run_summarization.py --mode=decode --data_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/dowsery-train.bin --vocab_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/vocab.bin --log_root=/home/Framework/Prototyp/tensorflow-text-summary/binary_data --exp_name=myexperiment


https://github.com/pltrdy/pointer-generator

"""
from colored import fg, bg, attr
import translitcodec
import unicodedata
import os
import sys, time
import MySQLdb as mdb
import codecs
import os
import re
import sys
import codecs
import string
import time
import glob
import getopt
import argparse
from unidecode import unidecode
from datetime import datetime as dTime
from pprint import PrettyPrinter

file_path = "/home/Framework/Prototyp/dowsery-demo/train_data/"
os.makedirs(file_path, exist_ok=True)

file_path1 = "/home/Framework/Prototyp/dowsery-demo/binary_data/"
os.makedirs(file_path1, exist_ok=True)


#SqlQuery = "SELECT DISTINCT * FROM openjurv4 WHERE 1=1 AND p_hasleitsatz=1 ORDER BY RAND() LIMIT 25;"; # db:dowery_prototype
SqlQuery = "SELECT DISTINCT * FROM openjurv4 WHERE 1=1 AND p_hasleitsatz=1 LIMIT 25000;"; # db:dowery_prototype

# https://conceptnet.s3.amazonaws.com/downloads/2017/numberbatch/numberbatch-17.06.txt.gz


... [truncated, 81 more lines] ...

"""Example of Converting TextSum model data.
Usage:
python textsum_data_convert.py --command text_to_binary --in_directories dailymail/stories --out_files dailymail-train.bin,dailymail-validation.bin,dailymail-test.bin --split 0.8,0.15,0.05
python textsum_data_convert.py --command text_to_vocabulary --in_directories cnn/stories,dailymail/stories --out_files vocab
"""

"""
python3 DowserySummaryPreprepareADV.py

python3 textsum_data_convert.py --command text_to_binary --in_directories train_data --out_files binary_data/dowsery-train.bin,binary_data/dowsery-validation.bin,binary_data/dowsery-test.bin --split 0.8,0.15,0.05

python3 textsum_data_convert.py --command text_to_vocabulary --in_directories train_data --out_files binary_data/vocab.bin

python3 pointer-generator/run_summarization.py --mode=train --data_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/dowsery-* --vocab_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/vocab.bin --log_root=/home/Framework/Prototyp/tensorflow-text-summary/binary_data --exp_name=myexperiment
"""
from DowseryLibrary import *
import collections
import struct
import sys

from os import listdir
from os.path import isfile, join

from nltk.tokenize import sent_tokenize

import tensorflow as tf
from tensorflow.core.example import example_pb2

from numpy.random import seed as random_seed
from numpy.random import shuffle as random_shuffle

random_seed(123)  # Reproducibility

FLAGS = tf.app.flags.FLAGS
tf.app.flags.DEFINE_string('command', 'text_to_binary',
                           'Either text_to_vocabulary or text_to_binary.'
                           'Specify FLAGS.in_directories accordingly.')
tf.app.flags.DEFINE_string('in_directories', '', 'path to directory')
tf.app.flags.DEFINE_string('out_files', '', 'comma separated paths to files')
tf.app.flags.DEFINE_string('split', '', 'comma separated fractions of data')

def _text_to_binary(input_directories, output_filenames, split_fractions):
  filenames = _get_filenames(input_directories)
  
  random_shuffle(filenames)
  
  start_from_index = 0
  for index, output_filename in enumerate(output_filenames):
    sample_count = int(len(filenames) * split_fractions[index])
    print(output_filename + ': ' + str(sample_count))

... [truncated, 81 more lines] ...