#!/usr/bin/perl

# author: thecerial
# generiere das zoozle news archive
# vorher:

	# rm *-2006.html
	# rm *,emule.html
	# rm *,torrent.html

system("renice 20 $$");
system("clear");

use LWP::UserAgent;
use HTTP::Cookies;
use HTTP::Request;
use LWP::Simple;
use IO::Handle;


my @months = (
	'January',
	'February',
	'March',
	'April',
	'May',
	'June',
	'July',
	'August',
	'September',
	'October',
	'November',
	'December'
);

my @monate = (
	"Januar",
	"Februar",
	"Maerz",
	"April",
	"Mai",
	"Juni",
	"Juli",
	"August",
	"September",
	"Oktober",
	"November",
	"Dezember"
);


#! unterscheidung in news und newsarchive

#my $StartUrlDE = "http://www.zoozle.net/emule-bittorrent-news-deutsch";
my $StartUrlEN = "http://www.zoozle.org/emule-bittorrent-news-english";

#my $PATHDE = "/server/wwwroot/emule-bittorrent-news-deutsch";
my $PATHEN = "/server/wwwroot/emule-bittorrent-news-english";

#my $ArchiveUrlDE = "http://www.zoozle.net/emule-bittorrent-newsarchive-deutsch";
my $ArchiveUrlEN = "http://www.zoozle.org/emule-bittorrent-newsarchive-english";
#my $ARCHIVEPATHDE = "/server/wwwroot/emule-bittorrent-newsarchive-deutsch";
my $ARCHIVEPATHEN = "/server/wwwroot/emule-bittorrent-newsarchive-english";

my ( $sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst ) = localtime(time);
$year = $year+1900;
$mon = $mon;

# http://www.zoozle.net/emule-bittorrent-news-english/4-january-2007.html
# http://www.zoozle.net/emule-bittorrent-news-english/4-january-2007,a-d,emule.html

#&GenerateCache( "$StartUrlDE", $PATHDE . '/index.html' );
#&GenerateCache( "$StartUrlEN", $PATHEN . '/index.html' );

my $count = 0;

## für 2006 ####
for ( $i=0; $i<=11; $i++ ) {		# monate
	for ( $j=1; $j<=31; $j++ ) {	# tage
		
		$dateDE		= @monate[$i];
		$dateEN		= @months[$i];
	
		$StringSimpleDE	= lc "$j-$dateDE-2006";
		$StringSimpleEN	= lc "$j-$dateEN-2006";
		
	#	&GenerateCache( "$StartUrlDE/$StringSimpleDE.html", $PATHDE );	
		&GenerateCache( "$StartUrlEN/$StringSimpleEN.html", $PATHEN );		# http://www.zoozle.net/emule-bittorrent-news-english/4-january-2007.html

		foreach ( "0-9", "a-d", "e-k", "l-n", "o-s", "t-z" )  {
			
			# http://www.zoozle.net/emule-bittorrent-news-english/4-january-2007,a-d,emule.html
		#	&GenerateCache( "$StartUrlDE/$StringSimpleDE,$_,torrent.html", $PATHDE );
			&GenerateCache( "$StartUrlEN/$StringSimpleEN,$_,torrent.html", $PATHEN );

		#	&GenerateCache( "$StartUrlDE/$StringSimpleDE,$_,emule.html", $PATHDE );
			&GenerateCache( "$StartUrlEN/$StringSimpleEN,$_,emule.html", $PATHEN );

		}; # foreach 

	}; # for ( $j=1; $j<=31; $j++ ) {	
}; # for ( $i=0; $i<=11; $i++ ) {		# monate
###Ende: 2006


######## für 2007 hier nur ##########
my $howmuch = 0;
for ( my $i=0; $i<=$mon; $i++ ) {		# monate
	
	# zähle immer nur bis zum aktuellen tag des monats und bereite dazu nur die sitemaps vor: sprich wenn jetzt grade januar der 21 ist, dann ist es sinnfrei die sitemap bis zum 31.1 schon zu erstellen
	if ( $i == $mon ) {
		$howmuch = $mday;
	} else {
		$howmuch = 31;
	};

	for ( my $j=1; $j<=$howmuch; $j++ ) {	# tage
		
		$dateDE		= @monate[$i];
		$dateEN		= @months[$i];
	
		$StringSimpleDE	= lc "$j-$dateDE-2007";
		$StringSimpleEN	= lc "$j-$dateEN-2007";
		
	#	&GenerateCache( "$StartUrlDE/$StringSimpleDE.html", $PATHDE );	
		&GenerateCache( "$StartUrlEN/$StringSimpleEN.html", $PATHEN );		# http://www.zoozle.net/emule-bittorrent-news-english/4-january-2007.html

		foreach ( "0-9", "a-d", "e-k", "l-n", "o-s", "t-z" )  {
			
			# http://www.zoozle.net/emule-bittorrent-news-english/4-january-2007,a-d,emule.html
		#	&GenerateCache( "$StartUrlDE/$StringSimpleDE,$_,torrent.html", $PATHDE );
			&GenerateCache( "$StartUrlEN/$StringSimpleEN,$_,torrent.html", $PATHEN );

		#	&GenerateCache( "$StartUrlDE/$StringSimpleDE,$_,emule.html", $PATHDE );
			&GenerateCache( "$StartUrlEN/$StringSimpleEN,$_,emule.html", $PATHEN );

		}; # foreach 

	}; # for ( $j=1; $j<=31; $j++ ) {	# tage
}; # for ( $i=0; $i<=12; $i++ ) {		# monate	
######## ENDE: für 2007 hier nur ##########



exit;


sub GenerateCache(){

	my $url			= shift;
	my $path		= shift;

	my $content		= &http_get( $url,60,'zoozle-newsbot','' );
	my (@PATH)		= split('/', $url);
	my $lenght		= sprintf("%.2f", length($content) / 1024);
	
	print "Bearbeite Eintrag $count: [ $lenght Kb ] $url\n"; # $path/$PATH[$#PATH] \n";
	unlink "$path/$PATH[$#PATH]";
	
	if ( length($content) >= 10000 ) {

		open(WH,">$path/$PATH[$#PATH]") or warn;
			#binmode WH, ":utf8";
			flock(WH, 2);
			WH->autoflush(1);
			print WH $content;
			print WH "<center style=\"font-size: 9px; color:#888888\">Cached Version from : " . MySQLDateTime() . "</center>";
		close WH;

		system("chmod 755 $path/$PATH[$#PATH]");

	}; # if ( length($content) >= 10000 ) {

	$count++;
	return 1;

}; # sub GenerateCache(){}


sub http_get() {
	
	my $url			= shift;
	my $timeout		= shift || 20;
	my $useragent	= shift || 'Zoozle-Caching-Bot 1.0';
	my $ProxyCFG	= shift;

	my $UA			= LWP::UserAgent->new( keep_alive => 1 );

	$UA->agent( $useragent );
	$UA->timeout( $timeout );
	$UA->max_size( 95000000 );	# hier: einbauen, wie groß die seite sein darf - no limit in how big the page can be

	# cookie_jar => {}    # an empty, memory-only HTTP::Cookies object
	my $jar_jar = HTTP::Cookies->new
		( file => "$ENV{HOME}/.SpiderCookies.txt" || ".SpiderCookies.txt",
			autosave => 1,
			max_cookie_size => 40960,
			max_cookies_per_domain => 10000, );
			$UA->cookie_jar( $jar_jar );

	my $req = HTTP::Request->new(GET => $url);
	
	if ( %{$ProxyCFG}->{"USEPROXY"} == 1 ) {
		$UA->proxy(['http', 'ftp'] => %{$ProxyCFG}->{"PROXYURL"});
		$req->proxy_authorization_basic( %{$ProxyCFG}->{"PROXYUSER"}, %{$ProxyCFG}->{"PROXYPASS"});
	} else {
		$req->proxy_authorization_basic( %{$ProxyCFG}->{"PROXYUSER"}, %{$ProxyCFG}->{"PROXYPASS"});
	};

	# Pass request to the user agent and get a response back
  	my $res = $UA->request($req);

	if ( $res->is_success ) {
		
		return $res->content;

   	};

	return 0;

}; # sub http_get {}
############# Subroutine zum erstellen eines HTTP Requestes:Ende


sub MySQLDateTime() {

	my ( $sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst ) = localtime(time);
	$year = $year+1900;
	$mon = $mon + 1;

	if ( length($mon) == 1 ) {
		$mon = "0". $mon;
	};
	if ( length($mday) == 1 ) {
		$mday = "0". $mday;
	};
	if ( length($sec) == 1 ) {
		$sec = "0". $sec;
	};
	if ( length($min) == 1 ) {
		$min = "0". $min;
	};
	if ( length($hour) == 1 ) {
		$hour = "0". $hour;
	};
	   
	return $year ."-". $mon ."-". $mday .' '. $hour .':'. $min .':'. $sec;
	
}; # sub MySQLDateTime() {}


sub MySQLDateTimeSimple() {

	my ( $sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst ) = localtime(time);
	$year = $year+1900;
	$mon = $mon + 1;

	if ( length($mon) == 1 ) {
		$mon = "0". $mon;
	};
	if ( length($mday) == 1 ) {
		$mday = "0". $mday;
	};
	if ( length($sec) == 1 ) {
		$sec = "0". $sec;
	};
	if ( length($min) == 1 ) {
		$min = "0". $min;
	};
	if ( length($hour) == 1 ) {
		$hour = "0". $hour;
	};
	   
	return $year ."-". $mon ."-". $mday;
	
}; # sub MySQLDateTime() {}


sub GenerateCache1OLD(){

	my $url			= shift;
	my $path		= shift;

	my $content		= &http_get( $url );
#	my $LinksARef	= &get_links( \$content, $url );
#	my $count = 0;

	foreach my $entry ( @{$LinksARef} ) {
		my (@PATH)=split('/', $entry);
		
		print "Bearbeite Eintrag $count: $PATH[$#PATH] \n" if ( $PATH[$#PATH] =~ /\.htm/) ;
		
		my $content1 = &http_get( $entry );
		open(WH,">$path/$PATH[$#PATH]") if ( $PATH[$#PATH] =~ /\.htm/) ;
			print WH $content;
			print WH "<center style=\"font-size: 9px; color:#888888\">Cached Version from : " . MySQLDateTime() . "</center>";
		close WH;

	}; # foreach my $entry ( @{$LinksARef} ) {}


}; # sub GenerateCache(){}

#http://www.zoozle.net/emule-bittorrent-newsarchive-deutsch/2006-02,31,de,news.html
#
#list mit urls generieren, mod rewrite umstellen auf anderen pfad, und im normalen pfad dann den cache lassen
#
#
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,0-9,de,bittorrent,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,a-d,de,bittorrent,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,e-k,de,bittorrent,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,l-n,de,bittorrent,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,o-s,de,bittorrent,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,t-z,de,bittorrent,archive.html
#
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,0-9,de,emule,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,a-d,de,emule,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,e-k,de,emule,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,l-n,de,emule,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,o-s,de,emule,archive.html
#http://www.zoozle.net/emule-bittorrent-news-deutsch/2006-02-25,t-z,de,emule,archive.html
