dowsery - dowsery-demo2 - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/env python
# https://developers.google.com/custom-search/docs/xml_results#countryCodes
#https://www.linkedin.com/countserv/count/share?format=jsonp&url=https://www.buzzerstar.com 
# pip install --upgrade spacy tensorflow gensim sumy keras markovify google-api-python-client beautifulsoup4
#from sphinxapi import *
import sys, time
import hashlib
import pickle
#import argparse
import json
#import sys

import re
import nltk
import os, sys, logging, re

from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from nltk.tokenize import PunktSentenceTokenizer
from nltk.stem.snowball import SnowballStemmer

#pip3 install stop-words
from stop_words import get_stop_words
stopwordsDE = get_stop_words('de')

from nltk.corpus import stopwords

cachedStopWords = stopwords.words("german")

# load nltk's English stopwords as variable called 'stopwords'
stopwords 	= nltk.corpus.stopwords.words('german')

debug				= False
max_gensim_results 	= 23

import os
import re
import sys
import codecs
import string
import time
import glob
import getopt
import argparse
from unidecode import unidecode

... [truncated, 767 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/env python
"""
python3 DowserySummaryPreprepareADV.py

python3 textsum_data_convert.py --command text_to_binary --in_directories train_data --out_files binary_data2/dowsery-train.bin,binary_data2/dowsery-validation.bin,binary_data2/dowsery-test.bin --split 1.0,0,0

python3 textsum_data_convert.py --command text_to_vocabulary --in_directories train_data --out_files binary_data2/vocab.bin

python3 pointer-generator/run_summarization.py --mode=train --data_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data2/dowsery-* --vocab_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data2/vocab.bin --log_root=/home/Framework/Prototyp/tensorflow-text-summary/binary_data2 --exp_name=myexperiment2

-> Single Pass
python3 pointer-generator/run_summarization.py --mode=decode --data_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/dowsery-train.bin --vocab_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/vocab.bin --log_root=/home/Framework/Prototyp/tensorflow-text-summary/binary_data --exp_name=myexperiment


https://github.com/pltrdy/pointer-generator

"""
from colored import fg, bg, attr
import translitcodec
import unicodedata
import os
import sys, time
import MySQLdb as mdb
import codecs
import os
import re
import sys
import codecs
import string
import time
import glob
import getopt
import argparse
from unidecode import unidecode
from datetime import datetime as dTime
from pprint import PrettyPrinter

file_path = "/home/Framework/Prototyp/dowsery-demo/train_data/"
os.makedirs(file_path, exist_ok=True)

file_path1 = "/home/Framework/Prototyp/dowsery-demo/binary_data/"
os.makedirs(file_path1, exist_ok=True)


#SqlQuery = "SELECT DISTINCT * FROM openjurv4 WHERE 1=1 AND p_hasleitsatz=1 ORDER BY RAND() LIMIT 25;"; # db:dowery_prototype
SqlQuery = "SELECT DISTINCT * FROM openjurv4 WHERE 1=1 AND p_hasleitsatz=1 LIMIT 25000;"; # db:dowery_prototype

# https://conceptnet.s3.amazonaws.com/downloads/2017/numberbatch/numberbatch-17.06.txt.gz


... [truncated, 81 more lines] ...

"""Example of Converting TextSum model data.
Usage:
python textsum_data_convert.py --command text_to_binary --in_directories dailymail/stories --out_files dailymail-train.bin,dailymail-validation.bin,dailymail-test.bin --split 0.8,0.15,0.05
python textsum_data_convert.py --command text_to_vocabulary --in_directories cnn/stories,dailymail/stories --out_files vocab
"""

"""
python3 DowserySummaryPreprepareADV.py

python3 textsum_data_convert.py --command text_to_binary --in_directories train_data --out_files binary_data/dowsery-train.bin,binary_data/dowsery-validation.bin,binary_data/dowsery-test.bin --split 0.8,0.15,0.05

python3 textsum_data_convert.py --command text_to_vocabulary --in_directories train_data --out_files binary_data/vocab.bin

python3 pointer-generator/run_summarization.py --mode=train --data_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/dowsery-* --vocab_path=/home/Framework/Prototyp/tensorflow-text-summary/binary_data/vocab.bin --log_root=/home/Framework/Prototyp/tensorflow-text-summary/binary_data --exp_name=myexperiment
"""
from DowseryLibrary import *
import collections
import struct
import sys

from os import listdir
from os.path import isfile, join

from nltk.tokenize import sent_tokenize

import tensorflow as tf
from tensorflow.core.example import example_pb2

from numpy.random import seed as random_seed
from numpy.random import shuffle as random_shuffle

random_seed(123)  # Reproducibility

FLAGS = tf.app.flags.FLAGS
tf.app.flags.DEFINE_string('command', 'text_to_binary',
                           'Either text_to_vocabulary or text_to_binary.'
                           'Specify FLAGS.in_directories accordingly.')
tf.app.flags.DEFINE_string('in_directories', '', 'path to directory')
tf.app.flags.DEFINE_string('out_files', '', 'comma separated paths to files')
tf.app.flags.DEFINE_string('split', '', 'comma separated fractions of data')

def _text_to_binary(input_directories, output_filenames, split_fractions):
  filenames = _get_filenames(input_directories)
  
  random_shuffle(filenames)
  
  start_from_index = 0
  for index, output_filename in enumerate(output_filenames):
    sample_count = int(len(filenames) * split_fractions[index])
    print(output_filename + ': ' + str(sample_count))

... [truncated, 81 more lines] ...