#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";
require "/root/zoozle/new-bots-X/functions.pl";

# renicing to 20 !
system("renice 20 $$");
system("clear");

# module laden
use strict;
no strict "refs";
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;
my $ScanningTimeout		= 30;

my $torrentflatfile		= "/root/zoozle/new-bots-X/torrent_news.txt";
unlink $torrentflatfile;

# snarf-it.org und slotorrent.net sind tot

my %LinksToScanDE = (		# 27 torrents
		"http://bitreactor.to/index"											=> '\/torrent\/',			# http://bitreactor.to/torrent/592734-filme-divx-xvid-otac-na-sluzbenom-putu-1985
		#"http://speedtorrent.to/index.php"										=> 'popup\.php\?Active=',	# http://speedtorrent.to/popup.php?Active=filme&ID=20310
		#"http://torrent.to/torrent.php"											=> 'torrent\.php\?Mod',	
		"http://www.saugstube-torrent.to/Go/News"								=> '\/Torrent\/',			# http://www.saugstube-torrent.to/saugstube.php?Module=Details&ID=173972
		"http://thepiratebay.org/search/deutsch/0/3/0"							=> '\/torrent\/',	
		"http://thepiratebay.org/search/german/0/3/0"							=> '\/torrent\/',
		"http://www.mininova.org/search/?search=german"							=> '\/tor\/',	
		"http://www.mininova.org/search/?search=deutsch"						=> '\/tor\/',	
		"http://www.torrentz.com/search?q=german&x=0&y=0"						=> '(\w{32})',	
		"http://www.torrentz.com/search?q=deutsch&x=0&y=0"						=> '(\w{32})',	
		"http://www.torrentportal.com/torrents-search.php?search=german"		=> '\/details\/',	
		"http://www.torrentportal.com/torrents-search.php?search=deutsch"		=> '\/details\/',	
		"http://btjunkie.org/search?q=german"									=> '\/torrent\/',	
		"http://btjunkie.org/search?q=deutsch"									=> '\/torrent\/',	
		"http://www.torrentreactor.net/search.php?search=&words=deutsch&cid=&sid=&type=1&exclude=&orderby=a.id&asc=0&x=53&y=7"	 => '\/torrents\/',	# http://www.torrentreactor.net/view.php?id=324165
		"http://www.torrentreactor.net/search.php?search=&words=german&cid=&sid=&type=1&exclude=&orderby=a.id&asc=0&x=53&y=7"	 => '\/torrents\/',	# http://www.torrentreactor.net/view.php?id=324165
		"http://www.torrentreactor.to/search.php?q=deutsch&submit.x=0&submit.y=0"					=> '\/torrents\/',	# http://www.torrentreactor.net/view.php?id=324165
		"http://www.torrentreactor.to/search.php?q=german&submit.x=0&submit.y=0"					=> '\/torrents\/',	# http://www.torrentreactor.net/view.php?id=324165
		"http://www.sumotorrent.com/searchResult.php?search=german"				=> 'details',	
		"http://www.sumotorrent.com/searchResult.php?search=deutsch"			=> 'details',	
		"http://www.btmon.com/torrent/german"									=> 'torrent\.html',	
		"http://www.btmon.com/torrent/deutsch"									=> 'torrent\.html',	
		"http://www.seedpeer.com/search.php?search=german"						=> '\/details\/',	
		"http://www.seedpeer.com/search.php?search=deutsch"						=> '\/details\/',	
		"http://www.fulldls.com/search-all-torrents/?qa=german&active=active"	 => 'torrent-',	
		"http://www.fulldls.com/search-all-torrents/?qa=deutsch&active=active"	=> 'torrent-',	
		"http://fenopy.com/?keyword=german&select=0&order=0&sort=0&minsize=&maxsize=&search.x=0&search.y=0&search=Search"	 => '\/torrent\/',	
		"http://fenopy.com/?keyword=deutsch&select=0&order=0&sort=0&minsize=&maxsize=&search.x=0&search.y=0&search=Search"	 => '\/torrent\/',	
		"http://www.mybittorrent.com/?keywords=german&x=0&y=0"					=> '\/torrent\/',	
		"http://www.mybittorrent.com/?keywords=deutsch&x=0&y=0"					=> '\/torrent\/',	
		"http://www.torrentbox.com/torrents-search.php?search=german&cat=0&submit=TBox+Search"	 => 'torrent_details\?id=',		# http://www.torrentbox.com/torrent_details?id=51139&hit=1
		"http://www.torrentbox.com/torrents-search.php?search=deutsch&cat=0&submit=TBox+Search"	 => 'torrent_details\?id=',		# http://www.torrentbox.com/torrent_details?id=51139&hit=1
		"http://www.titantorrent.to/admin/crawler.php?crawler=zoozle-8A83fsd9d&date=today&lang=de"	=> '-torrent-',		
		"http://www.titantorrent.to/admin/crawler.php?crawler=zoozle-8A83fsd9d&date=any&lang=de"	=> '-torrent-',		
		"http://www.vertor.com/?mod=search&words=deutsch&x=0&y=0&=&search="		=> '\/torrents\/(\d{3,})\/',	
		"http://www.vertor.com/?mod=search&words=german&x=0&y=0&=&search="		=> '\/torrents\/(\d{3,})\/',	
		"http://www.bushtorrent.com/torrents.php?search=&words=deutsch"			=> '\/torrents\/(\d{3,})\/',	
		"http://www.bushtorrent.com/torrents.php?search=&words=german"			=> '\/torrents\/(\d{3,})\/',
		"http://www.torrentmatrix.com/torrents-search.php?search=deutsch&incldead=0&inclexternal=0&cat=0&x=0&y=0" => 'torrents-details\.php\?id=',	
		"http://www.torrentmatrix.com/torrents-search.php?search=german&incldead=0&inclexternal=0&cat=0&x=0&y=0" => 'torrents-details\.php\?id=',	
		#""	 => '',	
);

my %LinksToScanFR = (

		"http://torrentmatrix.com/torrents-search.php?search=french&cat=0&cat=0&incldead=0&inclexternal=0"		=> 'torrents-details\.php\?id=',						# http://torrentmatrix.com/torrents-details.php?id=73347&hit
		"http://torrentmatrix.com/torrents-search.php?search=francais&cat=0&cat=0&incldead=0&inclexternal=0"		=> 'torrents-details\.php\?id=',						# http://torrentmatrix.com/torrents-details.php?id=73347&hit=1
		"http://www.torrentportal.com/torrents-search.php?search=francais"			=> '\/details\/',
		"http://www.torrentportal.com/torrents-search.php?search=french"			=> '\/details\/',
		"http://extratorrent.com/search/?new=1&search=francais&s_cat=0"				=> '\/torrent\/',
		"http://extratorrent.com/search/?new=1&search=french&s_cat=0"				=> '\/torrent\/',
		"http://thepiratebay.org/search/francais/0/0/0"								=> '\/torrent\/',
		"http://thepiratebay.org/search/french/0/0/0"								=> '\/torrent\/',
		"http://www.torrentreactor.net/search.php?search=&words=francais&cid=&sid=&type=1&exclude=&orderby=a.id&asc=0&x=53&y=7"	 => '\/torrents\/',
		"http://www.torrentreactor.net/search.php?search=&words=french&cid=&sid=&type=1&exclude=&orderby=a.id&asc=0&x=53&y=7"	 => '\/torrents\/',
		"http://www.torrentreactor.to/search.php?q=french&submit.x=0&submit.y=0"	 => '\/torrents\/',	# http://www.torrentreactor.net/view.php?id=324165
		"http://www.torrentreactor.to/search.php?q=francais&submit.x=0&submit.y=0"	 => '\/torrents\/',	# http://www.torrentreactor.net/view.php?id=324165
		"http://www.sumotorrent.com/searchResult.php?search=french"				=> 'details',	
		"http://www.sumotorrent.com/searchResult.php?search=francais"				=> 'details',	
		"http://www.btmon.com/torrent/french"									=> 'torrent\.html',	
		"http://www.btmon.com/torrent/francais"									=> 'torrent\.html',	
		"http://www.seedpeer.com/search.php?search=french"						=> '\/details\/',	
		"http://www.seedpeer.com/search.php?search=francais"					=> '\/details\/',	
		"http://www.fulldls.com/search-all-torrents/?qa=french&active=active"	 => 'torrent-',	
		"http://www.fulldls.com/search-all-torrents/?qa=francais&active=active"	=> 'torrent-',	
		"http://fenopy.com/?keyword=french&select=0&order=0&sort=0&minsize=&maxsize=&search.x=0&search.y=0&search=Search"	 => '\/torrent\/',	
		"http://fenopy.com/?keyword=francais&select=0&order=0&sort=0&minsize=&maxsize=&search.x=0&search.y=0&search=Search"	 => '\/torrent\/',	
		"http://www.mybittorrent.com/?keywords=french&x=0&y=0"					=> '\/torrent\/',	
		"http://www.mybittorrent.com/?keywords=francais&x=0&y=0"					=> '\/torrent\/',	
		"http://www.torrentbox.com/torrents-search.php?search=german&cat=0&submit=TBox+Search"	 => 'torrent_details\?id=',		# http://www.torrentbox.com/torrent_details?id=51139&hit=1
		"http://www.torrentbox.com/torrents-search.php?search=detusch&cat=0&submit=TBox+Search"	 => 'torrent_details\?id=',		# http://www.torrentbox.com/torrent_details?id=51139&hit=1
		"http://www.titantorrent.to/admin/crawler.php?crawler=zoozle-8A83fsd9d&date=today&lang=fr"	=> '-torrent-',		
		"http://www.titantorrent.to/admin/crawler.php?crawler=zoozle-8A83fsd9d&date=any&lang=fr"	=> '-torrent-',		
		"http://www.vertor.com/?mod=search&words=french&x=0&y=0&=&search="		=> '\/torrents\/(\d{3,})\/',	
		"http://www.vertor.com/?mod=search&words=francais&x=0&y=0&=&search="		=> '\/torrents\/(\d{3,})\/',	
		"http://www.bushtorrent.com/torrents.php?search=&words=french"			=> '\/torrents\/(\d{3,})\/',	
		"http://www.bushtorrent.com/torrents.php?search=&words=francais"			=> '\/torrents\/(\d{3,})\/',
		"http://www.torrentmatrix.com/torrents-search.php?search=french&incldead=0&inclexternal=0&cat=0&x=0&y=0" => 'torrents-details\.php\?id=',	
		"http://www.torrentmatrix.com/torrents-search.php?search=francais&incldead=0&inclexternal=0&cat=0&x=0&y=0" => 'torrents-details\.php\?id=',	
		
		#""	 => '',	
);

my %LinksToScanEN = (
		"http://www.torrentreactor.net/"	 									=> '\/torrents\/',
		"http://isohunt.com/latest.php?mode=bt"									=> '\/release\/',	
		"http://www.torrentfive.com/index.php"									=> '\/download\/',	
		"http://www.torrentbox.com/torrents-browse.php?news"					=> 'torrent_details\?id=',	
		"http://www.mybittorrent.com/"											=> '\/torrent\/',		
		"http://fenopy.com/today.html"											=> '\/torrent\/',	
		"http://www.fulldls.com/today.php"										=> 'torrent-',		
		"http://www.btmon.com/"													=> 'torrent\.html',	
		"http://www.sumotorrent.com/index.php?sort=today"						=> 'details',	
		"http://www.torrentreactor.net/index.php?action=today"					=> '\/torrents\/',	
		"http://www.torrentreactor.to/"											=> '\/torrents\/',	
		"http://btjunkie.org/?do=latest"										=> '\/torrent\/',	
		"http://www.torrentportal.com/new-torrents.php"							=> '\/details\/',		
		"http://www.torrentz.com/movies"										=> '(\w{32})',	
		"http://www.torrentz.com/tv"											=> '(\w{32})',	
		"http://www.torrentz.com/music"											=> '(\w{32})',	
		"http://www.torrentz.com/games"											=> '(\w{32})',	
		"http://www.mininova.org/today/"										=> '\/tor\/',		
		"http://bitreactor.to/search?q=english&cid=0&lid=all&submit=Suchen"		=> '\/torrent\/',			# http://bitreactor.to/torrent/592734-filme-divx-xvid-otac-na-sluzbenom-putu-1985
		"http://www.titantorrent.to/admin/crawler.php?crawler=zoozle-8A83fsd9d&date=today&lang=all"	=> '-torrent-',		
		"http://www.titantorrent.to/admin/crawler.php?crawler=zoozle-8A83fsd9d&date=any&lang=all"	=> '-torrent-',		
		"http://www.bushtorrent.com/"											=> '\/torrents\/(\d{3,})\/',
		"http://www.torrentmatrix.com/torrents.php"								=> 'torrents-details\.php\?id=',	
		"http://thepiratebay.org/recent"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/1"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/2"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/3"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/4"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/5"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/6"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/7"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/8"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/9"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/10"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/11"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/12"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/13"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/14"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/15"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/16"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/17"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/18"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/19"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/20"										=> '\/torrent\/',
		"http://thepiratebay.org/recent/21"										=> '\/torrent\/',
		"http://pirlog.com/verified.php"										=> '\/torrent\/(\d{1,})',
		"http://pirlog.com/verified.php?pageno=2"								=> '\/torrent\/(\d{1,})',
		"http://pirlog.com/verified.php?pageno=3"								=> '\/torrent\/(\d{1,})',
		"http://pirlog.com/verified.php?pageno=4"								=> '\/torrent\/(\d{1,})',
		"http://pirlog.com/verified.php?pageno=5"								=> '\/torrent\/(\d{1,})',
		"http://www.torrent.ws/"												=> '\/files\/',
	#""	 => '',	

);


Main();


open(WH,"+>>/root/spiderlog.txt");
my $Load = &LoadCheck();
print WH getHumanTime()." $0 Start Adding to SQL - $Load\n";

&AddContentToSqlTORRENT($torrentflatfile);

my $Load = &LoadCheck();
print WH getHumanTime()." $0 End Adding to SQL - $Load\n";
close WH;

sub Main(){

# nicht forken !

	while( my ($url,$regex) = each(%LinksToScanDE) ){
		
		eval {

			local $SIG{'ALRM'} = sub { die "timed out\n" };

			alarm($ScanningTimeout);
			&doScanning( $url, $regex, "de" );
			alarm(0);

		};
		
		if ($@) {
			if ($@ eq "timed out\n") {
					print "I timed out while scanning $url\n";  
			}
			else {
					print "something else went boom\n";
			}
		}
		else {
				print "I didn't time out\n";
		}

	}; # while( my ($url,$goto) = each(%LinksToScan) ){

	while( my ($url,$regex) = each(%LinksToScanFR) ){
		
		eval {

			local $SIG{'ALRM'} = sub { die "timed out\n" };

			alarm($ScanningTimeout);
			&doScanning( $url, $regex, "fr" );
			alarm(0);

		};
		
		if ($@) {
			if ($@ eq "timed out\n") {
					print "I timed out while scanning $url\n";  
			}
			else {
					print "something else went boom\n";
			}
		}
		else {
				print "I didn't time out\n";
		}

	}; # while( my ($url,$goto) = each(%LinksToScan) ){

	while( my ($url,$regex) = each(%LinksToScanEN) ){
		
		eval {

			local $SIG{'ALRM'} = sub { die "timed out\n" };

			alarm($ScanningTimeout);
			&doScanning( $url, $regex, "en" );
			alarm(0);

		};
		
		if ($@) {
			if ($@ eq "timed out\n") {
					print "I timed out while scanning $url\n";  
			}
			else {
					print "something else went boom\n";
			}
		}
		else {
				print "I didn't time out\n";
		}
	}; # while( my ($url,$goto) = each(%LinksToScan) ){

	
	sub doScanning(){

		my $UrltoScan		= shift;
		my $Regex			= shift;
		my $Lang			= shift;

		my $Links			= &ScanningPage($UrltoScan,,);
		#print Dumper $Links;

		open(WH,"+>>$torrentflatfile");
		foreach my $link ( @{$Links} ) {
		
			my ( $LinkText, $ParrentUrl, $UrlToScan1 ) = split(' ### ', $link );
			
			chomp($LinkText);
			chomp($ParrentUrl);
			chomp($UrlToScan1);
			
			if ( $UrlToScan1 =~ /$Regex/i && $UrlToScan1 =~ /^http/i ) {
			# if ( $UrlToScan1 =~ /^http/i && rindex($UrltoScan,$Regex) != -1 ) {
							
				$UrlToScan1 =~ s/\&hit=1//gi;
				$LinkText =~ s/(\.|\-|\+|_|�)/ /ig;
				$LinkText =~ s/(\d)\///ig;
				$LinkText =~ s/(\d)+$//ig;
				$LinkText =~ s/^\s+//;
				$LinkText =~ s/\s+$//;
								
				# �berspringe bereits gescannte linktexte
				next if ( exists $AlreadSeenLinks->{$LinkText} );
				$AlreadSeenLinks->{$LinkText} = 0;

				print WH "CATE=$Lang#DESC=$LinkText#LINK=$UrlToScan1\n";
				print "CATE=$Lang#DESC=$LinkText#LINK=$UrlToScan1\n";

			} else { 
				
				#print rindex($UrltoScan,$Regex);
				#print "\n";

			}; # if ( $UrlToScan1 =~ /$Regex/i  ) {

		}; # foreach my $link ( @{$ExtractedLinks} ) {
		close WH;

		%AlreadSeenLinks		= ();
		return 1;

	}; # sub doScanning(){

	return 1;

}; # sub Main(){

# gucken, ob morgen immer noch refused bei isohunt
# print &ScanningPage("http://www.zoozle.net/emule-bittorrent-download/gothic,torrent,,0.html",,);
# exit;


sub ScanningPage(){

	my $UrltoScan			= shift || return 0;
	my $ParrentUrl			= shift || "";
	my $LinkText			= shift || "";


	# bereits gescannte links nicht mehr scannen + keine forum links scannen
	return if ( exists $AlreadSeenLinks->{$UrltoScan} || $UrltoScan =~ /forum/ig );

	# markiere den link als bereits gescannt - tu das hierschon, weil wir sp�ter eine menge return anweisung haben
	# und wir ohne die anweisung hier, sp�ter eventuell zig links doppelt spidern w�rden
	$AlreadSeenLinks->{$UrltoScan} = 0;


	my $HeadHashResultRef	= $HEAD->Head( $UrltoScan );
	my $HeadStatusCode		= $HeadHashResultRef->{ 'STA' };
	my $ContentType			= $HeadHashResultRef->{ 'TYP' };
	my $ContentLength		= $HeadHashResultRef->{ 'LEN' };

	# splitte den status code auf, um sp�ter das erste zeichen davon herrauszuholen
	@StatusCode				= split('', $HeadHashResultRef->{ 'COD' } );

#	# kehre zur�ck, wenn kein g�ltiger header code ausgegeben wurde, der statuscode fehlerhaft ist oder content lenght zu klein ist
#	if ( $StatusCode[0] != 2 ){
#		print "something is wrong status '$StatusCode[0]'\n";
#		return;
#	} elsif ( $HeadStatusCode != 1 ){
#		print "something is wrong head '$HeadStatusCode'\n";
#		return;
#	};

	$AlreadyScannedLinks++;
	my $GetHashResultRef	= $GET->Get( $UrltoScan );
	my $ResponseContent		= $GetHashResultRef->{ 'CNT' };
	my $ResponseObj			= $GetHashResultRef->{ 'OBJ' };

	my $ScanLinkConfig = {
		"OBJ"	=> $ResponseObj,
		"CON"	=> $ResponseContent,
		"CDP"	=> 0,
		"SUD"	=> 1,
		"SDT"	=> 0,	# 0=CurrentDept | 1=dirdept
		"URL"	=> $UrltoScan,
		"PURL"	=> $ParrentUrl,
		"TXT"	=> $LinkText,
	};

#	use Data::Dumper;
#	print Dumper 

	return $LINK->HtmlLinkExtractor($ScanLinkConfig)
	
}; # sub ScanningPage(){




sub getHumanTime {
	my ($Sekunden, $Minuten, $Stunden, $Monatstag, $Monat,
    $Jahr, $Wochentag, $Jahrestag, $Sommerzeit) = localtime(time);
	my $CTIME_String = localtime(time);
	$Monat+=1;
	$Jahrestag+=1;
	$Monat = $Monat < 10 ? $Monat = "0".$Monat : $Monat;
	$Monatstag = $Monatstag < 10 ? $Monatstag = "0".$Monatstag : $Monatstag;
	$Stunden = $Stunden < 10 ? $Stunden = "0".$Stunden : $Stunden;
	$Minuten = $Minuten < 10 ? $Minuten = "0".$Minuten : $Minuten;
	$Sekunden = $Sekunden < 10 ? $Sekunden = "0".$Sekunden : $Sekunden;
	$Jahr+=1900;
	my @Wochentage = ("Sonntag","Montag","Dienstag","Mittwoch","Donnerstag","Freitag","Samstag");
	my @Monatsnamen = ("","Januar","Februar","M&auml;rz","April","Mai","Juni",
	           "Juli","August","September","Oktober","November","Dezember");
	my $finale="$Wochentage[$Wochentag], $Monatstag. $Monatsnamen[$Monat] $Jahr, $Stunden:$Minuten:$Sekunden";
	return $finale;	

}

sub LoadCheck(){

	open(LOAD,"/proc/loadavg");
		
		my $load_tmp = <LOAD>;
		my ($load1,$load2,$load3)=split(" ", $load_tmp);
	
	close LOAD;

	return "$load1,$load2,$load3";

}; # sub LoadCheck(){