#!/usr/bin/perl

#sleep 25000;

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: BSD
### Version: 1.1.b  - 20080716@19.Uhr
### Contact: sebastian.enger@gmail.com | icq: 135846444
### Latest Version: PENDRIVE:\Programming\Perl\IncrementalBackup
###########################

use strict;
use DBI;
use File::Find;	# perl -MCPAN -e 'force install "File::Find"'
use File::Copy;	# perl -MCPAN -e 'force install "File::Copy"'
use File::Path;	# perl -MCPAN -e 'force install "File::Path"'
#use XML::Code; # perl -MCPAN -e 'force install "Curses"'
use File::Basename;
use Data::Dumper;
use String::Random; # perl -MCPAN -e 'force install "String::Random"'
use Digest::SHA qw(sha512_hex);
use Encode qw(encode decode);
use HTML::LinkExtractor; # perl -MCPAN -e 'force install "HTML::LinkExtractor"'
# perl -MCPAN -e 'force install "URI"'
use Lingua::Identify qw(:language_identification); # perl -MCPAN -e 'force install "Lingua::Identify"'
 
my $scriptname 		= basename($0);
my $count 			= 0;
my $pass 			= new String::Random;
my $LX 				= new HTML::LinkExtractor(undef,undef,1);

my $SourceFolder 	= '/home/sumy/db/synonyme.dict2.de/deutsch';
my $string1;
my $catg;
my $title;
my $string;
my $rcount 			= 0;
my $Hash;
my $dbh;
my %Hash 			= ();

##system("mysqldump -u root -p###########!########### -f --skip-skip-lock-tables buzzerstar > buzzerstarv3.sql");exit;
my $enc = 'latin1'; # in dieser Kodierung ist das Script gespeichert

unlink "text/logfile_$scriptname.txt";
open(L,">text/logfile_$scriptname.txt");
binmode(L, ":utf8");

# prepare files from SourceFolder
#eval {
find(\&SourceStructure, $SourceFolder);
#};
close L;

print "I found $count entries with overall $rcount Synonyms\n";
#$dbh->disconnect();
exit;

sub SourceStructure(){

    # $_ is set to the current file name
    # $File::Find::dir is set to the current directory
    # $File::Find::name is set to "$File::Find::dir/$_"
    # you are chdir()'d to $File::Find::dir 

	my $SourceFilename		= $File::Find::name;
	my $SourceDirname		= $File::Find::dir;
	my $string = "";
	
	# if ( $SourceFilename =~ /\.html/ig && $SourceFilename !~ /trackback/ig){ #$SourceFilename =~ /\.(html|htm|shtm|shtml|php)/ig ){
	if ($SourceFilename){	
		{
		  local $/ = undef;
		  open FILE, "$SourceFilename" or die "Couldn't open file: $SourceFilename $!";
		  binmode FILE;
		  $string = <FILE>;
		  close FILE;
		}
		
		my $dbh = DBI->connect("dbi:mysql:dbname=onetipp","root","###########99", 
			{
				RaiseError => 0, PrintError => 1, mysql_enable_utf8 => 1
			}
		);
				
				my (@content_text) 		= ($string =~ m#<ol class="synonyms">\s*(.*?)\s*</ol>#igs);
				my $content_text_raw 	= join("",@content_text);
					
				my ($first, $syn_db_ret)	= &parse_synonyms_two($content_text_raw);
				
				next if $first 				!~ /(\w{2,})/ig;
				next if $syn_db_ret 		!~ /(\w{2,})/ig;
				
				#print "first: $first und syn: $syn_db_ret\n";
				#next;
				$count++;
				
				my $query = sprintf("%s (%s, %s, %s)",
					"INSERT INTO synonym_table (word,synonyms,uid) VALUES",
					$dbh->quote($first),
					$dbh->quote($syn_db_ret),		
					$dbh->quote(""),											
				);
				eval { $dbh->do($query) };
				$syn_db_ret = "";
			
				$dbh->disconnect;
				
	}; # if ( $SourceFilename =~ /\.txt/ig ){#&& $SourceDirname =~ /Team/ig){
	
}; # sub SourceStructure(){

sub parse_synonyms(){
	my $text_raw 	= shift;
	my $first_word	= shift;
	
	my $syn_db		= "";
	
	chomp($first_word);
	return if length($first_word) < 3;
		
	my ($language, $probability) = langof($first_word);
	#print "Language of ^$first_word^ is: $language with Wahrscheinlichkeit: $probability\n";
	next if $language ne "de";
	
	$LX->parse(\$text_raw);
		
	my $tmp_count = 0;
	for my $Link( @{ $LX->links } ) {
		
		my $synonym = $$Link{_TEXT};
		chomp($synonym);
		
		next if $synonym eq $first_word; # keine doppelten einträge
		next if $synonym =~ /standardsprachlich/igs;
		
		if ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
			$syn_db .= "$synonym;";
			$rcount++;
		}
	#	if($tmp_count == 0) {
	#	#	print "($language) -> First Word: $first_word -> Synonyms: $synonym; ";
	#		
	#	#	print L "First Word: $first_word -> Synonyms: $synonym; ";
	#		$rcount++;
	#		$tmp_count++;
	#	} elsif ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
	#	#	print "$synonym; ";
	#		
	#	#	print L "$synonym; ";
	#		$rcount++;
	#	}
	}
#	print L "\n";
#	print "\n\n";
	return $syn_db;
}

sub parse_synonyms_two(){
	my $text_raw 	= shift;
	my $syn_db		= "";
	
	my $first_word 	= "";
	
	$LX->parse(\$text_raw);
		
	my $tmp_count = 0;
	for my $Link( @{ $LX->links } ) {
		
		my $synonym = $$Link{_TEXT};
		chomp($synonym);
		
		if($tmp_count == 0) {
			$first_word = $synonym;
			$tmp_count++;
		}
		
		next if $synonym eq $first_word; # keine doppelten einträge
		next if $synonym =~ /standardsprachlich/igs;
		
		if ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
			$syn_db .= "$synonym;";
		}
		
	#	} elsif ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
	#	#	print "$synonym; ";
	#		
	#	#	print L "$synonym; ";
	#		$rcount++;
	#	}
	}
#	print L "\n";
#	print "\n\n";
	return ($first_word,$syn_db);
}

sub strip_tags(){
	my $content = shift;
	$content =~ s|<.+?>||ig;
	$content =~ s|content="||ig;
	$content =~ s|"||ig;
	return $content;
}
sub strip_tags_array(){
	my @arr = shift;
	my @retArray = ();
	foreach my $content (@arr){
	#my $content = shift;
	$content =~ s|<.+?>||ig;
	$content =~ s|.+?>||ig;
	$content =~ s|content="||ig;
	$content =~ s|"||ig;
	push(@retArray, $content);
	#$content =~ s#<([^">]+(?:"[^"]+")*[^>]+)>##ig;
	}
	return @retArray;
}
sub links(){
	my $content = shift;
	my @links = $content =~ /href\s*=\s*"?([^"\s>]+)/gis;
	return @links;
}
