#!/usr/bin/perl

#sleep 25000;

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: BSD
### Version: 1.1.b  - 20080716@19.Uhr
### Contact: sebastian.enger@gmail.com | icq: 135846444
### Latest Version: PENDRIVE:\Programming\Perl\IncrementalBackup
###########################

use strict;
use DBI;
use File::Find;	# perl -MCPAN -e 'force install "File::Find"'
use File::Copy;	# perl -MCPAN -e 'force install "File::Copy"'
use File::Path;	# perl -MCPAN -e 'force install "File::Path"'
#use XML::Code; # perl -MCPAN -e 'force install "Curses"'
use File::Basename;
use Data::Dumper;
use String::Random; # perl -MCPAN -e 'force install "String::Random"'
use Digest::SHA qw(sha512_hex);
use Encode qw(encode decode);
use HTML::LinkExtractor; # perl -MCPAN -e 'force install "HTML::LinkExtractor"'
# perl -MCPAN -e 'force install "URI"'
use Lingua::Identify qw(:language_identification); # perl -MCPAN -e 'force install "Lingua::Identify"'
 
my $scriptname 		= basename($0);
my $count 			= 0;
my $pass 			= new String::Random;
my $LX 				= new HTML::LinkExtractor(undef,undef,1);

#my $SourceFolder 	= '/home/sumy/db/synonyme.woxikon.de/synonyme/gut.php';
my $SourceFolder 	= '/home/sumy/db/www.woerterbuch.info/deutsch-englisch/synonym';
my $string;
my $rcount 			= 0;
my $Hash;
my $dbh;
my %Hash 			= ();

##system("mysqldump -u root -p###########!########### -f --skip-skip-lock-tables buzzerstar > buzzerstarv3.sql");exit;
my $enc = 'latin1'; # in dieser Kodierung ist das Script gespeichert

unlink "text/logfile_$scriptname.txt";
open(L,">text/logfile_$scriptname.txt");
binmode(L, ":$enc");

#eval {
find(\&SourceStructure, $SourceFolder);
#};
close L;

print "I found $count entries with overall $rcount Synonyms\n";
#$dbh->disconnect();
exit;

sub SourceStructure(){

    # $_ is set to the current file name
    # $File::Find::dir is set to the current directory
    # $File::Find::name is set to "$File::Find::dir/$_"
    # you are chdir()'d to $File::Find::dir 

	my $SourceFilename		= $File::Find::name;
	my $SourceDirname		= $File::Find::dir;
	my $string = "";
	
	# if ( $SourceFilename =~ /\.html/ig && $SourceFilename !~ /trackback/ig){ #$SourceFilename =~ /\.(html|htm|shtm|shtml|php)/ig ){
	if ( $SourceFilename =~ /./ig){	
				{
				  local $/ = undef;
				  open FILE, "$SourceFilename" or return "Couldn't open file: $SourceFilename $!";
				  binmode FILE;
				  $string = <FILE>;
				  close FILE;
				}
				
				
				my (@allcontent) 		= ($string =~ m#<td align="left" valign="top" class="standard" width="50%"><x1><b>Deutsch</b>\s*(.*?)\s*<td align="left" valign="top" class="standard" width="50%"><x1><b>Englisch</b></x1>#igs);
				my $content 			= join(" ",@allcontent);
				
				my $first;
				my (@first) 			= ($content =~ m#<em>\s*(.*?)\s*</em>#igs);
				my $first_word 			= join(" ",@first);
				
				my $content_text;
				my (@content_text) 		= ($content =~ m#<x1>\s*(.*?)\s*</div>#igs);
				my $content_text_raw 	= join(" ",@content_text);
				
				for (my $i=0; $i<=scalar(@first); $i++){
					&parse_synonyms($content_text[$i], $first[$i]);
				}
				$count++;
				next;
						
					my $dbh = DBI->connect("dbi:mysql:dbname=buzzerstar","root","", {
						AutoCommit => 1,#0=$dbh->commit(); 
						RaiseError => 0,
					});
					
					my $query = sprintf("%s (%s, %s, %s, %s, %s)",
						"INSERT INTO witze (p_text,p_catg,p_shortcode,p_hash,id) VALUES",
						$dbh->quote(),
						$dbh->quote(),
						$dbh->quote($pass->randpattern("CncCncCncCnC")),
						$dbh->quote(sha512_hex("witze_text")),		
						$dbh->quote(""),											
					);
				
					eval { $dbh->do($query) };
					$dbh->disconnect;
				
	}; # if ( $SourceFilename =~ /\.txt/ig ){#&& $SourceDirname =~ /Team/ig){
	
}; # sub SourceStructure(){

sub parse_synonyms(){
	my $text_raw 	= shift;
	my $first_word	= shift;
	
	chomp($first_word);
	return if length($first_word) < 3;
		
	my ($language, $probability) = langof($first_word);
	#print "Language of ^$first_word^ is: $language with Wahrscheinlichkeit: $probability\n";
	next if $language ne "de";
	
	$LX->parse(\$text_raw);
		
	my $tmp_count = 0;
	for my $Link( @{ $LX->links } ) {
		
		my $synonym = $$Link{_TEXT};
		chomp($synonym);
		
		next if $synonym eq $first_word; # keine doppelten einträge
		next if $synonym =~ /standardsprachlich/igs;
		
		if($tmp_count == 0) {
			print "($language) -> First Word: $first_word -> Synonyms: $synonym; ";
			
			print L "First Word: $first_word -> Synonyms: $synonym; ";
			$rcount++;
			$tmp_count++;
		} elsif ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
			print "$synonym; ";
			
			print L "$synonym; ";
			$rcount++;
		}
	}
	print L "\n";
	print "\n\n";
	return;
}
sub strip_tags(){
	my $content = shift;
	$content =~ s|<.+?>||ig;
	$content =~ s|content="||ig;
	$content =~ s|"||ig;
	return $content;
}
sub strip_tags_array(){
	my @arr = shift;
	my @retArray = ();
	foreach my $content (@arr){
	#my $content = shift;
	$content =~ s|<.+?>||ig;
	$content =~ s|.+?>||ig;
	$content =~ s|content="||ig;
	$content =~ s|"||ig;
	push(@retArray, $content);
	#$content =~ s#<([^">]+(?:"[^"]+")*[^>]+)>##ig;
	}
	return @retArray;
}
sub links(){
	my $content = shift;
	my @links = $content =~ /href\s*=\s*"?([^"\s>]+)/gis;
	return @links;
}
