#!/usr/bin/perl

#################
##### Author:	Sebastian Enger 
##### eMail:	sebastian.enger@gmail.com
##### Web:		http://www.sebastian-enger.com
##### Licence:	Its only mine!
#################


system("renice 20 $$");
system("clear");


# https://www.google.com/webmasters/sitemaps/addsitemap?siteUrl=http://www.zoozle.net/emule-bittorrent-download/&amp;hl=en&type=0&sitemapUrl0= PLUS SITEMAP URI

# first do a to install needed modules: perl -MCPAN -e 'force install "use HTTP::Date"'

use HTTP::Date;
use WWW::Google::SiteMap::Ping;
use URI::Escape qw(uri_escape);
use POSIX qw(strftime);
use POSIX;
use IO::Handle;
use strict;
use LWP::Simple;
use String::Approx 'amatch';

my $VERSION = "22.7.2007 - V.2";

my $StatementHandle;
my $SiteMapFile;
my $KeyCounts;
my @data = ();
my $SiteMap;
my $data;

my $CurrentDate			= getLastModTimeStamp();
my $StoreSiteMapPath	= '/server/wwwroot/emule-bittorrent-download';	# no tailing /
my $StoreSiteMapWebPath	= 'http://www.zoozle.org/emule-bittorrent-download';		# no tailing /
my $Stopwords			= "/server/wwwroot/lib/stopwordlist.txt";	# pre teen filter

################ START: porno stopword liste
my %PornStopWords	= ();
my $PornStopWords	= {};

# preteen stopword filter
open(RH,"<$Stopwords") or warn "file $Stopwords does not exists!\n";

	while (<RH>) {
		my $line = $_;
		my $line = lc($line);
		next if ( $line =~ /^#/);
		chomp($line);
		$line =~ s/^\s+//;
		$line =~ s/\s+$//;
		$PornStopWords->{$line} = 0;
	}; # while (<RH>) {

close RH;
################ END: porno stopword liste

my %TABLES = (
	"emule7"			=> "sitemap_emule_today",
	"torrent7"			=> "sitemap_torrent_today",
);

my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
my ($wkday,$month,$day,$time,$year)						 = split(/\s+/, localtime);
$mon = $mon+1; 
#$year = $year + 1900;

my $KeyWordStorePath		= '/server/querys/keywords';
my $CurrentMonthKeywordPath = "$KeyWordStorePath/$mon.$year";
# my $CurrentMonthKeywordPath = "$KeyWordStorePath/12.2006";

my $CurrentDate				= "$day.$mon.$year";	# für den jeweils aktuellsten Tag
# my $CurrentDate				= $ARGV[0];			# für manuellen Tag print "$CurrentDate\n"; exit;

my $en = "$CurrentMonthKeywordPath/$CurrentDate.en.txt";
my $de = "$CurrentMonthKeywordPath/$CurrentDate.de.txt";


#unlink "sitemapurls-today-keywords.html";

GenerateSiteMaps($en,'en');
GenerateSiteMaps($de,'de');

sub GenerateSiteMaps(){

	my $file = shift;
	my $lang = shift;

	my $sitemapUp = 0;
	my $count = 0;

	my ( @Raw )		= split('/', $file);
	my $Raw			= @Raw[$#Raw];

	$Raw =~ s/\.en\.txt//i if ( $lang eq 'en' );
	$Raw =~ s/\.de\.txt//i if ( $lang eq 'de' );
	my ( $day,$mon,$year )	= split('\.', $Raw);
	
	if ( length($mon) == 1 ) {
		$mon = "0$mon";
	};

	if ( length($day) == 1 ) {
		$day = "0$day";
	};

	my $SqlDate				= $year ."-". $mon ."-". $day;
	
	my $DateTimestamp = str2time("$year-$mon-$day"); # enthält den unix timestamp vom angegebenen datum
	print "Install $file as date $DateTimestamp with lang $lang\n";
		
	open(RH,"<$file") or die;
	flock(RH, 2);
	     		
	$SiteMapFile	= "$StoreSiteMapPath/bittorrent.$SqlDate.$lang.$sitemapUp.xml";
	open(SITEMAP,">$SiteMapFile");
		binmode SITEMAP, ":utf8";
		flock(SITEMAP, 2);
	SITEMAP->autoflush(1);
	print SITEMAP PreXMLHeader();

	print "Generating: $SiteMapFile \n";

	my $LinkCount = 0;
	my $lastmod	  = getLastModTimeStamp($DateTimestamp);

	while( <RH> ) {

		my $IsForbiddenKwFlag	= &CheckPornStopwordList( $_ );
		next					if ( $IsForbiddenKwFlag == 1 );	# preeteen filter mit fuzzy match

		my $data = uri_escape($_);
				
		print SITEMAP "<url><loc>http://www.zoozle.org/emule-bittorrent-download/$data,torrent,en,0.html</loc><lastmod>$lastmod</lastmod><changefreq>daily</changefreq><priority>0.90</priority></url>\n";
				
		if ( rand >= 0.2 ) {
			print SITEMAP "<url><loc>http://www.zoozle.org/emule-bittorrent-download/$data,emule,en,0.html</loc><lastmod>$lastmod</lastmod><changefreq>daily</changefreq><priority>0.90</priority></url>\n";
			$LinkCount++;
		};

		$LinkCount++;
		
		if ( $LinkCount >= 20000 ) {
		
			print "$LinkCount Links Added\n";
			# close sitemap
			print SITEMAP SuffixXMLHeader();
			close SITEMAP;
			
			print "\tCompressing: $SiteMapFile.gz\n";

			# compressing file
			system("/usr/bin/gzip -f -b 9 $SiteMapFile");
			system("/bin/chmod 755 $SiteMapFile.gz");
			PingGoogleForSitemap("$StoreSiteMapWebPath/bittorrent.$SqlDate.$lang.$sitemapUp.xml.gz");
													
			# start new sitemap
			$sitemapUp++;
			$LinkCount = 0;
			
			$SiteMapFile	= "$StoreSiteMapPath/bittorrent.$SqlDate.$lang.$sitemapUp.xml";
			print "Generating: $SiteMapFile \n";

			open(SITEMAP,">$SiteMapFile");
				binmode SITEMAP, ":utf8";
				flock(SITEMAP, 2);
			SITEMAP->autoflush(1);
			print SITEMAP PreXMLHeader();

			print "Generating: $SiteMapFile \n";
			print SITEMAP "<url><loc>http://www.zoozle.org/emule-bittorrent-download/$data,torrent,en,0.html</loc><lastmod>$lastmod</lastmod><changefreq>daily</changefreq><priority>0.90</priority></url>\n";
	
			if ( rand >= 0.7 ) {
				print SITEMAP "<url><loc>http://www.zoozle.org/emule-bittorrent-download/$data,emule,en,0.html</loc><lastmod>$lastmod</lastmod><changefreq>daily</changefreq><priority>0.90</priority></url>\n";
				$LinkCount++;
			};

		}; # } elsif ( $KeyCounts <= 0 ) { 

	}; # while( <RH> ) {

	print "\tCompressing: $SiteMapFile.gz\n";
	# compressing file
	system("/usr/bin/gzip -f -b 9 $SiteMapFile");
	system("/bin/chmod 755 $SiteMapFile.gz");
	PingGoogleForSitemap("$StoreSiteMapWebPath/bittorrent.$SqlDate.$lang.$sitemapUp.xml.gz");

	$sitemapUp = 0;
	close RH;

}; # sub GenerateSiteMaps(){


sub PingGoogleForSitemap(){

	my $SiteMapUrl = shift;	

	open(WH,">>sitemapurls-today-keywords.html");
		print WH "<a href=\"https://www.google.com/webmasters/sitemaps/addsitemap?siteUrl=http://www.zoozle.org/emule-bittorrent-download/&amp;hl=en&type=0&sitemapUrl0=$SiteMapUrl\">$SiteMapUrl</a><br><br>\n";
	close WH;
  
	my $ping = WWW::Google::SiteMap::Ping->new(
		"$SiteMapUrl",
	);

	$ping->submit;
	# print "\tThese pings succeeded:\n";
	foreach($ping->success) {
		print "\tSUCESS: $_ \n" if ( $_ =~ /\w/ );
		print "$_\n";
	}

	# print "\tThese pings failed:\n";
	foreach($ping->failure) {
		print "\t\tFAILURE $_\n" if ( $_ =~ /\w/ );
		print "$_\n";
	}

	my $yahoo		= get("http://www.yahoo.com/ping?sitemap=$SiteMapUrl");
	my $msn			= get("http://search.msn.com/ping?sitemap=$SiteMapUrl");
	my $msn1		= get("http://search.live.com/ping?sitemap=$SiteMapUrl");
	my $yahoo1		= get("https://siteexplorer.search.yahoo.com/submit/ping?sitemap=$SiteMapUrl");
	   
	return 1;

}; # sub PingGoogleForSitemap

sub PreXMLHeader(){

	my $returnString = <<"END";
<?xml version="1.0" encoding="UTF-8"?>
<!-- generator="wordpress/2.0.4" -->
<!-- sitemap-generator-url="http://www.zoozle.net/" sitemap-generator-version="3.0b1" -->
<!-- generated-on="19. October 2006 20:21" -->
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.google.com/schemas/sitemap/0.84 http://www.google.com/schemas/sitemap/0.84/sitemap.xsd">

END
	
	return $returnString;

}; # sub PreXMLHeader(){


sub SuffixXMLHeader(){

	return "</urlset>";

}; # sub SuffixXMLHeader(){


sub getLastModTimeStamp(){

	my $now = shift || time();

	# We need to munge the timezone indicator to add a colon between the hour and minute part
	my $tz = strftime("%z", localtime($now));
	$tz =~ s/(\d{2})(\d{2})/$1:$2/;

	# ISO8601
	return  strftime("%Y-%m-%dT%H:%M:%S", localtime($now)) . "+00:00\n";
	
	# simple version, because WWW::Google::SiteMap ads tail automatically
	# return  strftime("%Y-%m-%d", localtime($now));

}; # sub getLastModTimeStamp(){


sub CheckPornStopwordList(){
			
	my $value	= shift;
	my @matched = ();

	# gehe alle stopwörter durch, und teste diese mittels fuzzy matching: String::Approx 'amatch'
	while ( my($key,$val) = each(%{$PornStopWords}) ){
		if ( length($key) >= 3 && length($value) >= 3 ) {
			my $matched = amatch($key, "90%", $value);
			if ( $value =~ /$key/ig ) {
			#	print "\t\t[$matched] $value regex matches $key\n";
				push(@matched, "$matched");
			}; # if ( $value =~ /$key/ig ) {
		}; #if ( length($key) >= 3 && length($value) >= 3 ) {
	}; # while( my($key,$val) = each(%PornStopWords) ){
	
	foreach my $match (@matched) {
		#my ($match,$key) = split('#',$match);
		if ( $match == 1 ) {
		#	print "Matched StopWord '$key'\n";
			return 1;
		}; # if ( $matched == 1 ) {
	}; # foreach my $match (@matched) {
	
	return 0;

}; # sub CheckPornStopwordList(){

exit;