#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";
require "/root/zoozle/new-bots-X/functions.pl";

# renicing to 20 !
system("renice 20 $$");
system("clear");

# module laden
use strict;
no strict "refs";
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;


my $torrentflatfile		= "/root/zoozle/new-bots-X/torrent_news.txt";
unlink $torrentflatfile;

# filedonkey emule

my %LinksToScanDE = (		# 27 torrents
		"http://bitreactor.to/index"											=> '\/torrent\/',			# http://bitreactor.to/torrent/592734-filme-divx-xvid-otac-na-sluzbenom-putu-1985
		"http://deluxebits.to/deluxebits/index.php"							=> 'details\.php\?id=',		# http://deluxebits.to/deluxebits/details.php?id=227592
		"http://www.torrents-and-more.to/pml/index.php"						=> '\-torrent\.html',		# http://www.torrents-and-more.to/pml/47241-Nicht-Auflegen-torrent.html
		"http://speedtorrent.to/index.php"									=> 'popup\.php\?Active=',	# http://speedtorrent.to/popup.php?Active=filme&ID=20310
		"http://torrent.to/torrent/"											=> 'popup\.php\?Active=',	
		"http://www.saugstube-torrent.to/saugstube.php"						=> 'saugstube\.php\?Module=Details\&ID=',		# http://www.saugstube-torrent.to/saugstube.php?Module=Details&ID=173972
		"http://downloadz.to/torrent/index.php"								=> 'popup\.php\?Active=',
		"http://www.torrentspy.com/search?query=german&submit.x=0&submit.y=0"	=> '\/torrent\/',			# http://www.torrentspy.com/torrent/1163336/www_torrent_to_Spiderman_3_TS_LD_German_XviD
		"http://www.torrentspy.com/search?query=deutsch&submit.x=0&submit.y=0"	=> '\/torrent\/',			# http://www.torrentspy.com/torrent/1163336/www_torrent_to_Spiderman_3_TS_LD_German_XviD
		"http://thepiratebay.org/search/deutsch/0/3/0"							=> '\/tor\/',	
		"http://thepiratebay.org/search/german/0/3/0"							=> '\/tor\/',
		"http://lister.mystik-torrent.net/"									=> 'popup\.php\?Active=',	
		"http://www.mininova.org/search/?search=german"						=> '\/tor\/',	
		"http://www.mininova.org/search/?search=deutsch"						=> '\/tor\/',	
		"http://www.torrentz.com/search?q=german&x=0&y=0"						=> '(\w{32})',	
		"http://www.torrentz.com/search?q=deutsch&x=0&y=0"						=> '(\w{32})',	
		"http://www.torrentportal.com/torrents-search.php?search=german"		=> '\/details\/',	
		"http://www.torrentportal.com/torrents-search.php?search=deutsch"		=> '\/details\/',	
		"http://btjunkie.org/search?q=german"									=> '\/torrent\/',	
		"http://btjunkie.org/search?q=deutsch"									=> '\/torrent\/',	
		"http://www.torrentreactor.net/search.php?search=&words=german&cid=0"	 => 'view\.php\?id=',	# http://www.torrentreactor.net/view.php?id=324165
		"http://www.torrentreactor.net/search.php?search=&words=deutsch&cid=0"	 => 'view\.php\?id=',	# http://www.torrentreactor.net/view.php?id=324165
		"http://lister.torrent-galaxy.to/"										 => '\/torrent\/details\/',	
		"http://www.snarf-it.org/pages/search.html?query=deutsch"				=> '\/viewTorrent\/',	
		"http://www.snarf-it.org/pages/search.html?query=german"				=> '\/viewTorrent\/',	
		"http://www.sumotorrent.com/searchResult.php?search=german"				=> 'details_',	
		"http://www.sumotorrent.com/searchResult.php?search=deutsch"				=> 'details_',	
		"http://www.btswarm.org/search/?q=german"								=> '(\w{32})',	
		"http://www.btswarm.org/search/?q=deutsch"								=> '(\w{32})',	
		"http://www.btmon.com/torrent/german"									=> 'torrent\.html',	
		"http://www.btmon.com/torrent/deutsch"									=> 'torrent\.html',	
		"http://www.slotorrent.net/search.html?search=german"					=> 'tor_',	
		"http://www.slotorrent.net/search.html?search=deutsch"					=> 'tor_',	
		"http://www.meganova.org/search.php?search=german"						=> '\/details\/',	
		"http://www.meganova.org/search.php?search=deutsch"					=> '\/details\/',	
		"http://www.fulldls.com/search-all-torrents/?qa=german&active=active"	 => 'downloading_',	
		"http://www.fulldls.com/search-all-torrents/?qa=deutsch&active=active"	 => 'downloading_',	
		"http://fenopy.com/?keyword=german&select=0&order=0&sort=0&minsize=&maxsize=&search.x=0&search.y=0&search=Search"	 => '\/torrent\/',	
		"http://fenopy.com/?keyword=deutsch&select=0&order=0&sort=0&minsize=&maxsize=&search.x=0&search.y=0&search=Search"	 => '\/torrent\/',	
		"http://www.wtorrent.com/search.aspx?pageno=1&keyword=german"			=> 'btinfo\.aspx\?bt_id=',		# http://www.wtorrent.com/btinfo.aspx?bt_id=446969
		"http://www.wtorrent.com/search.aspx?pageno=1&keyword=deutsch"			=> 'btinfo\.aspx\?bt_id=',		# http://www.wtorrent.com/btinfo.aspx?bt_id=446969
		"http://www.mybittorrent.com/?keywords=german&x=0&y=0"					=> '\/info\/',	
		"http://www.mybittorrent.com/?keywords=deutsch&x=0&y=0"					=> '\/info\/',	
		"http://www.torrentbox.com/torrents-search.php?search=german&cat=0&submit=TBox+Search"	 => 'torrent_details\?id=',		# http://www.torrentbox.com/torrent_details?id=51139&hit=1
		 "http://www.torrentbox.com/torrents-search.php?search=detusch&cat=0&submit=TBox+Search"	 => 'torrent_details\?id=',		# http://www.torrentbox.com/torrent_details?id=51139&hit=1
		#""	 => '',	
);

my %LinksToScanFR = (

		"http://torrentmatrix.com/torrents-search.php?search=french&cat=0&cat=0&incldead=0&inclexternal=0"		=> 'torrents-details\.php\?id=',						# http://torrentmatrix.com/torrents-details.php?id=73347&hit=1
		"http://all-torrent.net/"																				=> 'index.php\?option=com_torrenttrader\&action=view',	# http://all-torrent.net/index.php?option=com_torrenttrader&action=view&id=351&Itemid=28
		"http://www.torrentportal.com/torrents-search.php?search=francais"										=> '\/details\/',
		"http://extratorrent.com/search/?new=1&search=francais&s_cat=0"											=> '\/torrent\/',
		"http://thepiratebay.org/search/francais/0/0/0"															=> '\/tor\/',
		"http://torrentspy.com/search?query=francais&submit.x=0&submit.y=0"										=> '\/torrent\/',
			
		#""	 => '',	
);

my %LinksToScanEN = (
		
		"http://isohunt.com/latest.php?mode=bt"								=> '\/release\/',	
		"http://www.torrentfive.com/index.php"									=> '\/download\/',	
		"http://www.torrentbox.com/torrents-browse.php?news"					=> 'torrent_details\?id=',	
		"http://www.mybittorrent.com/"											=> '\/info\/',		
		"http://www.wtorrent.com/"												=> 'btinfo\.aspx\?bt_id=',
		"http://fenopy.com/today.html"											=> '\/torrent\/',	
		"http://www.fulldls.com/today.php"										=> 'downloading_',	
		"http://www.meganova.org/order-date.html"								=> '\/details\/',	
		"http://www.slotorrent.net/today.html"									=> 'tor_',	
		"http://www.btmon.com/"												=> 'torrent\.html',	
		"http://www.btswarm.org/search/?q=english"								=> '(\w{32})',	
		"http://www.btswarm.org/search/?q=englisch"							=> '(\w{32})',	
		"http://www.sumotorrent.com/index.php?sort=today"						=> 'details_',	
		"http://www.snarf-it.org/view/today.html?cat=Movies"					=> '\/viewTorrent\/',	
		"http://www.torrentreactor.net/index.php?action=today"					=> '\/torrents\/',	
		"http://btjunkie.org/?do=latest"										=> '\/torrent\/',	
		"http://www.torrentportal.com/new-torrents.php"						=> '\/details\/',		
		"http://www.torrentz.com/torrents"										=> '(\w{32})',	
		"http://www.mininova.org/today/"										=> '\/tor\/',		
		"http://bitreactor.to/search?q=english&cid=0&lid=all&submit=Suchen"	=> '\/torrent\/',			# http://bitreactor.to/torrent/592734-filme-divx-xvid-otac-na-sluzbenom-putu-1985
		"http://www.torrents-and-more.to/pml/search.php?term=english&cat=0"	=> '\-torrent\.html',		# http://www.torrents-and-more.to/pml/47241-Nicht-Auflegen-torrent.html
		"http://www.torrentspy.com/latest.asp"									=> '\/torrent\/',			# http://www.torrentspy.com/torrent/1163336/www_torrent_to_Spiderman_3_TS_LD_German_XviD
		"http://www.torrentspy.com/latest.asp?mode=yesterday"					=> '\/torrent\/',			# http://www.torrentspy.com/torrent/1163336/www_torrent_to_Spiderman_3_TS_LD_German_XviD
		"http://thepiratebay.org/recent"										=> '\/tor\/',
		"http://thepiratebay.org/recent/1"										=> '\/tor\/',
		"http://thepiratebay.org/recent/2"										=> '\/tor\/',
		"http://thepiratebay.org/recent/3"										=> '\/tor\/',
		"http://thepiratebay.org/recent/4"										=> '\/tor\/',
		"http://thepiratebay.org/recent/5"										=> '\/tor\/',
		"http://thepiratebay.org/recent/6"										=> '\/tor\/',
		"http://thepiratebay.org/recent/7"										=> '\/tor\/',
		"http://thepiratebay.org/recent/8"										=> '\/tor\/',
		"http://thepiratebay.org/recent/9"										=> '\/tor\/',
		"http://thepiratebay.org/recent/10"										=> '\/tor\/',
		"http://thepiratebay.org/recent/11"										=> '\/tor\/',
		"http://thepiratebay.org/recent/12"										=> '\/tor\/',
		"http://thepiratebay.org/recent/13"										=> '\/tor\/',
		"http://thepiratebay.org/recent/14"										=> '\/tor\/',
		"http://thepiratebay.org/recent/15"										=> '\/tor\/',
		"http://thepiratebay.org/recent/16"										=> '\/tor\/',
		"http://thepiratebay.org/recent/17"										=> '\/tor\/',
		"http://thepiratebay.org/recent/18"										=> '\/tor\/',
		"http://thepiratebay.org/recent/19"										=> '\/tor\/',
		"http://thepiratebay.org/recent/20"										=> '\/tor\/',
		"http://thepiratebay.org/recent/21"										=> '\/tor\/',
		#""	 => '',	

);


Main();
#&getNewsSex();
#&AddContentToSqlTORRENT($torrentflatfile);

sub Main(){

# nicht forken !

		my $url = "http://www.mininova.org/today/";
		my $regex = '\/tor\/';
#	while( my ($url,$regex) = each(%LinksToScanEN) ){
		&doScanning( $url, $regex, "en" );
#	}; # while( my ($url,$goto) = each(%LinksToScan) ){

	
	sub doScanning(){

		my $UrltoScan		= shift;
		my $Regex			= shift;
		my $Lang			= shift;

		my $Links			= &ScanningPage($UrltoScan,,);
		#print Dumper $Links;

		open(WH,"+>>tmp.txt");
		foreach my $link ( @{$Links} ) {
		
			my ( $LinkText, $ParrentUrl, $UrlToScan1 ) = split(' ### ', $link );
			
			chomp($LinkText);
			chomp($ParrentUrl);
			chomp($UrlToScan1);
			
			if ( $UrlToScan1 =~ /$Regex/i && $UrlToScan1 =~ /^http/i ) {
			# if ( $UrlToScan1 =~ /^http/i && rindex($UrltoScan,$Regex) != -1 ) {
							
				$UrlToScan1 =~ s/\&hit=1//gi;
				$LinkText =~ s/(\.|\-|\+|_|»)/ /ig;
				$LinkText =~ s/(\d)\///ig;
				$LinkText =~ s/(\d)+$//ig;
				$LinkText =~ s/^\s+//;
				$LinkText =~ s/\s+$//;
								
				# überspringe bereits gescannte linktexte
				next if ( exists $AlreadSeenLinks->{$LinkText} );
				$AlreadSeenLinks->{$LinkText} = 0;

				print WH "CATE=$Lang#DESC=$LinkText#LINK=$UrlToScan1\n";
				print "CATE=$Lang#DESC=$LinkText#LINK=$UrlToScan1\n";

			} else { 
				
				#print rindex($UrltoScan,$Regex);
				#print "\n";

			}; # if ( $UrlToScan1 =~ /$Regex/i  ) {

		}; # foreach my $link ( @{$ExtractedLinks} ) {
		close WH;

		%AlreadSeenLinks		= ();
		return 1;

	}; # sub doScanning(){

	return 1;

}; # sub Main(){

# gucken, ob morgen immer noch refused bei isohunt
# print &ScanningPage("http://www.zoozle.net/emule-bittorrent-download/gothic,torrent,,0.html",,);
# exit;


sub ScanningPage(){

	my $UrltoScan			= shift || return 0;
	my $ParrentUrl			= shift || "";
	my $LinkText			= shift || "";


	# bereits gescannte links nicht mehr scannen + keine forum links scannen
	return if ( exists $AlreadSeenLinks->{$UrltoScan} || $UrltoScan =~ /forum/ig );

	# markiere den link als bereits gescannt - tu das hierschon, weil wir später eine menge return anweisung haben
	# und wir ohne die anweisung hier, später eventuell zig links doppelt spidern würden
	$AlreadSeenLinks->{$UrltoScan} = 0;


	my $HeadHashResultRef	= $HEAD->Head( $UrltoScan );
	my $HeadStatusCode		= $HeadHashResultRef->{ 'STA' };
	my $ContentType			= $HeadHashResultRef->{ 'TYP' };
	my $ContentLength		= $HeadHashResultRef->{ 'LEN' };

	# splitte den status code auf, um später das erste zeichen davon herrauszuholen
	@StatusCode				= split('', $HeadHashResultRef->{ 'COD' } );

#	# kehre zurück, wenn kein gültiger header code ausgegeben wurde, der statuscode fehlerhaft ist oder content lenght zu klein ist
#	if ( $StatusCode[0] != 2 ){
#		print "something is wrong status '$StatusCode[0]'\n";
#		return;
#	} elsif ( $HeadStatusCode != 1 ){
#		print "something is wrong head '$HeadStatusCode'\n";
#		return;
#	};

	$AlreadyScannedLinks++;
	my $GetHashResultRef	= $GET->Get( $UrltoScan );
	my $ResponseContent		= $GetHashResultRef->{ 'CNT' };
	my $ResponseObj			= $GetHashResultRef->{ 'OBJ' };

	my $ScanLinkConfig = {
		"OBJ"	=> $ResponseObj,
		"CON"	=> $ResponseContent,
		"CDP"	=> 0,
		"SUD"	=> 1,
		"SDT"	=> 0,	# 0=CurrentDept | 1=dirdept
		"URL"	=> $UrltoScan,
		"PURL"	=> $ParrentUrl,
		"TXT"	=> $LinkText,
	};

#	use Data::Dumper;
#	print Dumper 

	return $LINK->HtmlLinkExtractor($ScanLinkConfig)
	
}; # sub ScanningPage(){



sub getNewsSex(){
	use LWP::Simple;

	open(WH,"+>>$torrentflatfile");
	my $page = get("http://sextorrent.to/index_in.php");
	my @CONTENT = split(/a href=\"/i, $page);
	foreach my $line ( @CONTENT ) {
		my $count++;
		if ( $line =~ /file\.php\?id=/i ) {	# http://sextorrent.to/file.php?id=13911&name=Mandy%20on%20Tour
			$line =~ s/amp;//i;

			my ( $link, undef ) = split('"', $line);
			my ( undef, $desc )		= split(/&name=/i, $link );
			
		#	print "LINK=$link und $desc\n";
			chomp($desc);
			next if ( $link =~ /java/ig );
			if ( ($desc =~ /(\w)/i) && ( $line =~ /(\w)/ ) ) {
				print WH "CATE=de#DESC=$desc#LINK=http://sextorrent.to/$link\n";
			};
		}; # if $line
	}; # foreach
	
	close WH;
	return 1;

}; # sub getNewsSex(){


# dead:	"http://dl-torrent.com/"																		=> 'torrents\.php\?mode=details\&id=',					# http://dl-torrent.com/torrents.php?mode=details&id=fee0d8b4588effcf5007ee7ca78f14336697df7a
	# DEAD: "http://francepartage.com/upload/browse.php?page=1"												=> 'details\.php\?id=',									# http://francepartage.com/upload/details.php?id=814

	#	"http://torrentmatrix.com/torrents-search.php?search=french&cat=0&cat=0&incldead=0&inclexternal=0"		=> 'torrents-details\.php\?id=',						# http://torrentmatrix.com/torrents-details.php?id=73347&hit=1
	#	"http://all-torrent.net/"																				=> 'index.php\?option=com_torrenttrader\&action=view',	# http://all-torrent.net/index.php?option=com_torrenttrader&action=view&id=351&Itemid=28
	#	"http://www.torrentportal.com/torrents-search.php?search=francais"										=> '\/details\/',
	#	"http://extratorrent.com/search/?new=1&search=francais&s_cat=0"											=> '\/torrent\/',
	#	"http://thepiratebay.org/search/francais/0/0/0"															=> '\/tor\/',
	#	"http://torrentspy.com/search?query=francais&submit.x=0&submit.y=0"										=> '\/torrent\/',
		
	






