#!/usr/bin/perl

#################
##### Author:	Sebastian Enger 
##### eMail:	sebastian.enger@gmail.com
##### Web:		http://www.sebastian-enger.com
##### Licence:	Its only mine!
#################

system("renice 20 $$");
system("clear");


###
# Crontab 
# env EDITOR=joe crontab -e
# 45 23 * * *  /usr/bin/perl /opt/pornrage/sitemaps/PornrageSitemapToday.pl
###

# https://www.google.com/webmasters/sitemaps/addsitemap?siteUrl=http://www.zoozle.net/emule-bittorrent-download/&amp;hl=en&type=0&sitemapUrl0= PLUS SITEMAP URI
# first do a to install needed modules: perl -MCPAN -e 'force install "WWW::Google::SiteMap::Ping"'
#    * http://www.yahoo.com/ping?sitemap=http%3A%2F%2Fjoseph.randomnetworks.com%2Fsitemap.xml
#    * http://www.google.com/ping?sitemap=http%3A%2F%2Fjoseph.randomnetworks.com%2Fsitemap.xml
#    * http://search.msn.com/ping?sitemap=http%3A%2F%2Fjoseph.randomnetworks.com%2Fsitemap.xml

# perl -MCPAN -e 'install "Digest::SHA1"'
#use URI::Escape qw(uri_escape);
use Search::Sitemap::Pinger::Ask;
use Search::Sitemap::Pinger::Yahoo;
use Search::Sitemap::Pinger::Google;
use Search::Sitemap::Pinger::Live;	# perl -MCPAN -e 'install "Search::Sitemap::Pinger"'
use Search::Sitemap::Ping;
use Search::Sitemap;
use POSIX qw(strftime);
use String::Approx 'amatch';		# perl -MCPAN -e 'install "String::Approx"'
use IO::Handle;
use strict;
use LWP::Simple;
use DBI;
use Net::MySQL;	# perl -MCPAN -e 'force install "Net::MySQL"'


my $drh	= DBI->install_driver("mysql");

my $DBHOST = "localhost";
my $DBNAME = "torrentrage";
my $DBUSER = "root";	
my $DBPASS = "iseeanape!23";# mysql root localhost: "zoozle23!99";

my $VERSION = "6.2.2009 - V.3";

my $StatementHandle;
my $SiteMapFile;
my $KeyCounts;
my @data = ();
my @sitemaps = ();
my $SiteMap;
my $data;
my $prio;

my $FilterMultiply		= 0.33; # für den filter
my $CurrentDate			= getLastModTimeStamp();
my $StoreSiteMapPath	= '/server/wwwroot/www.torrentrage.com/sitemaps';	# no tailing /
my $StoreSiteMapWebPath	= 'http://www.torrentrage.com/sitemaps';		# no tailing /
my $Stopwords			= "/server/wwwroot/www.torrentrage.com/lib/stopwordlist.txt";	# pre teen filter



################ START: porno stopword liste
my %PornStopWords	= ();
my $PornStopWords	= {};
my @PornStopWords	= ();

# preteen stopword filter
open(RH,"<$Stopwords") or warn "file $Stopwords does not exists!\n";

	while (<RH>) {
		my $line = $_;
		my $line = lc($line);
		next if ( $line =~ /^#/);
		chomp($line);
		$line =~ s/^\s+//;
		$line =~ s/\s+$//;
		$PornStopWords->{$line} = 0;
		push @PornStopWords, $line;
	}; # while (<RH>) {

close RH;
################ END: porno stopword liste


&GenerateSiteMaps( "pornragesitemaps" );
&GenerateSitemapIndexFile();


sub GenerateSitemapIndexFile(){

	open(W,">/server/wwwroot/www.torrentrage.com/sitemap.xml");
	print W <<EOF;
		<?xml version="1.0" encoding="UTF-8"?>
		<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
EOF
	foreach my $entry (@sitemaps){
		print W "<sitemap><loc>$entry</loc></sitemap>\n";
	};
	
	print W <<EOF;
   </sitemapindex>
EOF

	close W;
	
	#PingGoogleForSitemap("http://www.torrentrage.com/sitemap.xml");
	return 1;

}; # sub GenerateSitemapIndexFile(){


sub GenerateSiteMaps(){

	my $today			= shift;
	my $category		= shift;

	my $sitemapNumber	= 0;
	my $count			= 0;
	my $SiteMapFile		= "$StoreSiteMapPath/torrentrage-allsitemaps-news-$today-$sitemapNumber.xml";

	
	open(SITEMAP,"+>$SiteMapFile") or die "$!\n";
		binmode SITEMAP, ":utf8";
		flock(SITEMAP, 2);
	SITEMAP->autoflush(1);
	print SITEMAP PreXMLHeader();

	print "Generating: $SiteMapFile \n";

	my $dbh;
	my $StatementHandle;
	my $sth_count;
	my $KeyCounts;

	$dbh	= DBI->connect("DBI:mysql:database=$DBNAME:host=$DBHOST", "$DBUSER", "$DBPASS", {'RaiseError' => 0});
	$StatementHandle = $dbh->prepare("SELECT `DESC` FROM torrentrage;");
	$StatementHandle->execute();
	
	my $LinkCount = 0;
	my $lastmod	  = getLastModTimeStamp();
	my $DESC_last = "";
	my $len_last  = 0;

	while( @data = $StatementHandle->fetchrow_array()  ) {
		
		foreach my $data ( @data ) {
			
			$data = lc($data);
			$data = &trim_spaces($data);
					
			# $data is the description of on entry
			# my $IsForbiddenKwFlag	= &CheckPornStopwordList( $data );
			# next if ( $IsForbiddenKwFlag == 1 );
			next if ( length($data) <= 3 );

			# Mod begin
			my $DESC		= $data;
			my $len			= length($DESC);
			$len			= sprintf("%.0f", $len*$FilterMultiply );

			if ( $len_last != 0 && ( substr(lc($DESC), 0, $len) eq substr(lc($DESC_last), 0, $len) || substr(lc($DESC), 0, $len_last) eq substr(lc($DESC_last), 0, $len_last) )) {
				next;
			}; # if ( $len_last != 0 ){
					
			$DESC_last = $DESC;
			$len_last = $len;
			#Mod end

			my $prio	= &getSitempaPrio();
			my $rageurl = "http://www.torrentrage.com/torrent/$data.html";
			print SITEMAP "<url><loc>$rageurl</loc><lastmod>$lastmod</lastmod><changefreq>daily</changefreq><priority>$prio</priority></url>\n";
			
			$LinkCount++;
			
			if ( $LinkCount >= 45000 ) {
			
				print "$LinkCount Links Added\n";
				# close sitemap
				print SITEMAP SuffixXMLHeader();
				close SITEMAP;
				
				print "\tCompressing: $SiteMapFile.gz\n";

				# compressing file
				system("/usr/bin/gzip -f -b 9 $SiteMapFile");
				system("/bin/chmod 755 $SiteMapFile.gz");
			###	PingGoogleForSitemap("$StoreSiteMapWebPath/$SiteMapFile.gz");
				push(@sitemaps, "$StoreSiteMapWebPath/torrentrage-allsitemaps-news-$today-$sitemapNumber.xml.gz");
				
				
				# start new sitemap
				$sitemapNumber++;
				
				$SiteMapFile		= "$StoreSiteMapPath/torrentrage-allsitemaps-news-$today-$sitemapNumber.xml";
				print "Generating: $SiteMapFile \n";

				open(SITEMAP,">$SiteMapFile");
					binmode SITEMAP, ":utf8";
					flock(SITEMAP, 2);
				SITEMAP->autoflush(1);
				print SITEMAP PreXMLHeader();

				my $prio = &getSitempaPrio();

				print "Generating: $SiteMapFile \n";
				$LinkCount = 0;
	
			}; # } elsif ( $KeyCounts <= 0 ) { 

		}; # foreach my $data ( @data ) {
		
		@data = ();

	}; # while( $sth->fetch() ) {

	
	print SITEMAP SuffixXMLHeader();
	close SITEMAP;
	
	print "\tCompressing: $SiteMapFile.gz\n";
	# compressing file
	system("/usr/bin/gzip -f -b 9 $SiteMapFile");
	system("/bin/chmod 755 $SiteMapFile.gz");
	
###	PingGoogleForSitemap("$StoreSiteMapWebPath/torrentrage-allsitemaps-news-$today-$sitemapNumber.xml.gz");
	push(@sitemaps, "$StoreSiteMapWebPath/torrentrage-allsitemaps-news-$today-$sitemapNumber.xml.gz");
	
	$StatementHandle->finish();
	
	eval {
		$dbh->finish();
		$dbh->disconnect();
	};
	$sitemapNumber = 0;

}; # sub GenerateSiteMaps(){



sub PingGoogleForSitemap(){

	my $SiteMapUrl = shift;	

	my $ping = Search::Sitemap::Ping->new(
		$SiteMapUrl,
	);
  
	$ping->submit;
  
	for my $url ( $ping->urls ) {
	  print "$url\n";
	  for my $engine ( $ping->engines ) {
		  printf( "    %25s\n", $engine );
	  }
	}

	return 1;

}; # sub PingGoogleForSitemap


sub getSitempaPrio(){

	my $rand = substr(rand(0.10), 0, 4);
	my $rand2 = int(rand(8)+1);
	$prio = "0.$rand2" + $rand;
		
	if ( $prio >= 1 ) {
		$prio = 0.9;
	};

	return $prio;

}; # sub getSitempaPrio(){



sub PreXMLHeader(){

	my $returnString = <<"END";
<?xml version="1.0" encoding="UTF-8"?>
<!-- generator="wordpress/2.0.4" -->
<!-- sitemap-generator-url="http://www.zoozle.net/" sitemap-generator-version="3.0b1" -->
<!-- generated-on="19. October 2006 20:21" -->
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.google.com/schemas/sitemap/0.84 http://www.google.com/schemas/sitemap/0.84/sitemap.xsd">

END
	
	return $returnString;

}; # sub PreXMLHeader(){



sub SuffixXMLHeader(){

	return "</urlset>";

}; # sub SuffixXMLHeader(){



sub getLastModTimeStamp(){

	my $now	= time();

	# We need to munge the timezone indicator to add a colon between the hour and minute part
	my $tz = strftime("%z", localtime($now));
	$tz =~ s/(\d{2})(\d{2})/$1:$2/;

	# ISO8601
	return  strftime("%Y-%m-%dT%H:%M:%S", localtime($now)) . "+00:00\n";
	
	# simple version, because WWW::Google::SiteMap ads tail automatically
	# return  strftime("%Y-%m-%d", localtime($now));

}; # sub getLastModTimeStamp(){



sub CheckPornStopwordList(){
			
	my $value	= shift;
		
	my $split	= &CheckPornStopwordListSplit( $value );
	my $fuzzy	= &CheckPornStopwordListFuzzy( $value );

	# print "[$split] $fuzzy $value \n";
	# sleep 1;

	if ( $split > 1 || $fuzzy >= 1 ) {
		
		### print "BADWORD '$value' blocked\n";
		return 1;

	} else {

		return 0; # if matches == 0

	}; # if ( $matches != 0 || $fuzzy == 1 ) {

}; # sub CheckPornStopwordList(){


sub CheckPornStopwordListSplit(){
		
	my $value	= shift;

	for ( @PornStopWords ) {
	
		my @Link	= ();
		@Link		= eval{ split(/$_/ig, $value); };	# eval nicht vergessen
		my $matches	= @Link;

		# print "[$matches] [$_] $value - @Link\n";
		return $matches if ( $matches > 1 );

	}; # for ( @PornStopWords ) {

	return 0;

}; # sub CheckPornStopwordListSplit(){



sub CheckPornStopwordListFuzzy(){
			
	my $value	= shift;
	my @matched = ();

	# gehe alle stopwörter durch, und teste diese mittels fuzzy matching: String::Approx 'amatch'
	while ( my($key,$val) = each(%{$PornStopWords}) ){
		if ( length($key) >= 1 && length($value) >= 1 ) {
			my $matched = amatch($key, "90%", $value);
			if ( $value =~ /$key/ig ) {
			#	print "\t\t[$matched] $value regex matches $key\n";
				push(@matched, "$matched\n");
			}; # if ( $value =~ /$key/ig ) {
		}; #if ( length($key) >= 3 && length($value) >= 3 ) {
	}; # while( my($key,$val) = each(%PornStopWords) ){
	
#	foreach my $match (@matched) {
#		# my ($match,$key) = split('#',$match);
#		if ( $match == 1 ) {
#		#	print "Matched StopWord '$key'\n";
#			return 1;
#		}; # if ( $matched == 1 ) {
#	}; # foreach my $match (@matched) {
	
	for ( @matched ) {
		return $_ if ( $_ == 1 );
	}; # for ( @matched ) {

	return 0;

}; # sub CheckPornStopwordListFuzzy(){


sub trim_spaces(){

	my $title	= shift;
	
	$title =~ s/(\!|\?|\<|\>|\#|\@|\:|\)|\(|\||\*|\_|\]|\[)/+/gi;
	$title = &deleteSpecialChars($title);
	$title =~ s/[^[:ascii:]]//ig;
	$title =~ s/[^\w\d\s+\+]//gi; 

	$title =~ s/ +/+/ig;
	$title =~ s/\s+/+/ig;
	$title =~ s/\t+/+/ig;
	$title =~ s/\+{1,}/+/ig;

	my $content;
	my @content = split("", $title );

	$content[0] =~ s/[^[:ascii:]]//ig;
	
	if ( $content[0] eq '+' ){
		$content[0] = "";
	}
	if ( $content[$#content] eq '+' ){
		$content[$#content] = "";
	};

	return join("", @content);

}; # sub trim_spaces(){


sub deleteSpecialChars() {

	my $del_badchar = shift;

	if ( length($del_badchar) > 255 ) {
		# lösche alles nach dem 200sten zeichen bei überlangen eingaben
		$del_badchar = substr($del_badchar, 0, 255);		
	};

	$del_badchar =~ s/\"/\+/ig;
	$del_badchar =~ s/`/\+/ig;
	$del_badchar =~ s/\'/\+/ig;
	$del_badchar =~ s/\?/\+/ig;
	$del_badchar =~ s/%/\+/ig;
	$del_badchar =~ s/$/\+/ig;
	$del_badchar =~ s/§/\+/ig;
	$del_badchar =~ s/!/\+/ig;
	$del_badchar =~ s/\&/\+/ig;
	$del_badchar =~ s/\{/\+/ig;
	$del_badchar =~ s/\}/\+/ig;
	$del_badchar =~ s/\(/\+/ig;
	$del_badchar =~ s/\)/\+/ig;
	$del_badchar =~ s/\[/\+/ig;
	$del_badchar =~ s/\]/\+/ig;
	$del_badchar =~ s/=/\+/ig;
	$del_badchar =~ s/\#/\+/ig;
	$del_badchar =~ s/,/\+/ig;
	$del_badchar =~ s/\/\+/ig;
	$del_badchar =~ str_replace("|","+",$del_badchar);
	$del_badchar =~ s/</\+/ig;
	$del_badchar =~ s/>/\+/ig;
	$del_badchar =~ s/\//\+/ig;
	$del_badchar =~ s/°/\+/ig;
	$del_badchar =~ s/^/\+/ig;
	$del_badchar =~ s/\*/\+/ig;
	$del_badchar =~ s/\./\+/ig;
	$del_badchar =~ s/,/\+/ig;
	$del_badchar =~ s/ß/\+/ig;
	$del_badchar =~ s/\|/\+/ig;
	$del_badchar =~ s/€/\+/ig;
	$del_badchar =~ s/´/\+/ig;
	$del_badchar =~ s/~/\+/ig;
	$del_badchar =~ s/µ/\+/ig;

	
	return $del_badchar;

	# print str_replace('Hello','Goodbye',$string);  #Prints 'Goodbye World'

	sub str_replace
	{
		my $search = shift;
		my $replace = shift;
		my $subject = shift;
		if (! defined $subject) { return -1; }
		my $count = shift;
		if (! defined $count) { $count = -1; }
		
		my ($i,$pos) = (0,0);
		while ( (my $idx = index( $subject, $search, $pos )) != -1 )
		{
			substr( $subject, $idx, length($search) ) = $replace;
			$pos=$idx+length($replace);
			if ($count>0 && ++$i>=$count) { last; }
		}
		
		return $subject;
	}


}; # function deleteSpecialChars($del_badchar) {



exit;