#!/usr/bin/perl -Imodules

#########################################
##### Author:		Sebastian Enger / B.Sc
##### CopyRight:	Sebastian Enger
##### LastModified	16.07.2006
##### Function:		
##### Todo:			
########################################

#############
###### run from path: /root/zoozle/indexer/GenerellSpider
#############

# spider ignoriert forum und blog urls


# perl -MCPAN -e 'force install "XML::TreePP"'
# perl -MCPAN -e 'force install "APR::URI"'
# perl -MCPAN -e 'force install "Crypt::SSLeay"'

system("clear");

# link desc von javascript links holen
# lade immer alle module -> error checking

use strict;
use IO::Handle;
use Data::Dumper;
use TorrentIndexer::HTTP::Get;
use TorrentIndexer::HTTP::Head;
use TorrentIndexer::LINK::Extractor;
use TorrentIndexer::CHECKSUM::CheckSum;

use TorrentIndexer::TIME::Time;
use TorrentIndexer::HTML::HtmlParser;
use TorrentIndexer::CRYPTO::CryptoLibrary;
use TorrentIndexer::CRYPTO::SecureHashGenerator;
use TorrentIndexer::HTTP::RobotRules;
use TorrentIndexer::LINK::CurrentDept;
use TorrentIndexer::LINK::IsSameLink;
use TorrentIndexer::BITTORRENT::Bittorrent;


my @LinksToScan			= ();

my $CachePath			= "/root/zoozle/indexer/torrentindexer/cache";		# later check if dir exits if not create, siehe client.pm easyspider
mkdir $CachePath;					

my $ScanUntilDept		= 5;
my $CurrentDept		    = 0;

my $ScanDeptType		= 0;	# 0=CurrentDept | 1=diredept
my $StayOnSameHost		= 1;
my $AlreadyScannedFlag	= 0;


###########
### objekte initialisieren
###########


my $HEAD		= TorrentIndexer::HTTP::Head->new();
my $GET			= TorrentIndexer::HTTP::Get->new();
my $LINK		= TorrentIndexer::LINK::Extractor->new();
my $SAME		= TorrentIndexer::LINK::IsSameLink->new();
# my $BITTORRENT	= TorrentIndexer::BITTORRENT::Bittorrent->new();


#########
### MAIN
#########

my $CatLanguage = "en";
# my $StartUrl = "http://fenopy.com/";
# my $StartUrl = "http://www.torrentportal.com/";
# my $StartUrl = "http://www.mininova.org";
# my $StartUrl = "http://torrent.to/torrent/";
# my $StartUrl = "http://www.torrentreactor.net/";
# my $StartUrl = "http://www.mybittorrent.com/";
###my $StartUrl = "http://www.demonoid.com/";
my $StartUrl = "http://www.btmon.com/";

my (undef,undef,$Hostname,undef) = split('/', $StartUrl);

my $TorrentStoreFlatFile	= "$CachePath/$Hostname-torrent.txt";
my $LinkScannedFile			= "$CachePath/$Hostname-alreadyscannedlinks.txt"; 
my $LinksToScan				= "$CachePath/$Hostname-linksToScan.txt"; 


# todo: wenn man externen links folgen soll, dann müssen für die externen links auch neue TorrentFlatFiles usw angelegt werden


MainRoutine( $StartUrl, '', '' );


sub MainRoutine( ){

	my $CurrentUrlToScan	= shift;
	my $ParentLink			= shift;

	&Scanner( $CurrentUrlToScan, $ParentLink, '' );

	open(LINKS,"<$LinksToScan") or die "Cannot open $LinksToScan for writing: $!\n";
		LINKS->blocking(0);
		binmode LINKS, ":utf8";
		
	while(<LINKS>){
		my ( $LinkDesc, $ParentLink, $LinkName ) = split(' ### ', $_);
		
		chomp($LinkDesc); chomp($ParentLink); chomp($LinkName);

		# mininova: s/get/tor/;
		# print "LINK FOUND: $LinkName \n";
		# next;

		open(SCANNED,"<$LinkScannedFile") or die;
			
			SCANNED->autoflush(1);
			SCANNED->blocking(0);
			flock(SCANNED, 2);
			
			while(<SCANNED>){
				chomp($_);
				# DEBUG: print "'$_' und '$LinkName'\n"; 
				if ( lc($_) eq lc($LinkName) ) {
					$AlreadyScannedFlag = 1;
					next;
				}; # if 
			}; # while(<SCANNED>){

		close SCANNED;
	
		if ( $AlreadyScannedFlag == 0 ) {
			
			# wenn wir auf dem MainHost bleiben sollen
			if ( $StayOnSameHost == 1 ) {
			
				# vergleiche start url mit aktueller url aus $LinkName
				my $IsSameHostName = $SAME->IsSameLink( $StartUrl, $LinkName );
				# print "\t[ $IsSameHostName ] VERLGEICHE: $StartUrl, $LinkName\n";
								
				if ( $IsSameHostName == 1 ) {
					
					# print "Now i would scan: $LinkName \n";
					# print "[$IsSameHostName] VERLGEICHE: $ParentUrl, $LinkName\n";
					# sleep 1;
					
					&Scanner( $LinkName, $ParentLink, $LinkDesc );
									
				} else {
					
					# wenn der host nicht gleich der mainurl ist, dann markiere den link als gescanned
					# $AlreadyScannedLinks{$LinkName}++;

				}; # if ( $IsSameHostName == 1 ) {

			} else { 
				
				# wenn wir nicht auf dem mainurl bleiben sollen, sondern wandern dürfen
				#  &scanne $HashRef->{'LINKNAME'}
				# $AlreadyScannedLinks{$LinkName}++;

			}; #ENDE: if ( $StayOnSameHost == 1 ) {

		} else { # if ( $AlreadyScannedFlag == 1 ) {

			# print "ALREADY SCANNED :[COUNT] " . $AlreadyScannedLinks{$LinkName} . " for LINK $LinkName \n";
			# print "\t ALREADY SCANNED $LinkName \n";
			# sleep 1;

		}; # if ( exists($AlreadySc

		# $AlreadyScannedFlag wieder auf Null setzen
		$AlreadyScannedFlag = 0;

	}; # while(<LINKS>){

	close LINKS;

}; # sub MainRoutine(){


sub Scanner(){

	my $CurrentUrlToScan = shift;
	my $ParentLink		 = shift;
	my $CurrentLinkDesc	 = shift;

	print "SCANNE: '$CurrentUrlToScan'\n";

	my $HeadHashResultRef	= $HEAD->Head($CurrentUrlToScan);
	my $HeadStatusCode		= $HeadHashResultRef->{ 'STA' };
	
	# print Dumper($HeadHashResultRef);
	# print "\n";

	if ( $HeadStatusCode == 1 ) {

		my $ContentType			= $HeadHashResultRef->{ 'TYP' };
		my $GetHashResultRef	= $GET->Get($CurrentUrlToScan);
		
		my $ResponseObj			= $GetHashResultRef->{ 'OBJ' };
		my $ResponseContent		= $GetHashResultRef->{ 'CNT' };
		my $CurrentCurrentDept		= 0;

		# print "ContentTyp=$ContentType\n";
		# select(undef,undef,undef,0.1);

		if ( $ContentType =~ /html/i ) {	# wenn $CurrentUrlToScan html datei ist

			my $ScanLinkConfig = {
				"OBJ" => $ResponseObj,
				"CON" => $ResponseContent,
				"CDP" => $CurrentDept,
				"SUD" => $ScanUntilDept,
				"SDT" => $ScanDeptType,
				"URL" => $CurrentUrlToScan,
			};

			$LINK->HtmlLinkExtractor( $ScanLinkConfig, $LinksToScan );
			#push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );		
		
		} elsif ( $ContentType =~ /xml/i ) { # wenn $CurrentUrlToScan xml datei ist
			# or if $CurrentUrlToScan =~ rss|xml|feed
			
			my $ScanLinkConfig = {
				"OBJ" => $ResponseObj,
				"CON" => $ResponseContent,
				"CDP" => $CurrentDept,
				"SUD" => $ScanUntilDept,
				"SDT" => $ScanDeptType,
				"URL" => $CurrentUrlToScan,
			};

			$LINK->XmlLinkExtractor( $ScanLinkConfig, $LinksToScan );
			#push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );	

		} elsif ( $ContentType =~ /plain/i ) {  # wenn $CurrentUrlToScan text datei ist

			my $ScanLinkConfig = {
				"OBJ" => $ResponseObj,
				"CON" => $ResponseContent,
				"CDP" => $CurrentDept,
				"SUD" => $ScanUntilDept,
				"URL" => $CurrentUrlToScan,
				"SDT" => $ScanDeptType,
			};

			$LINK->TextLinkExtractor( $ScanLinkConfig, $LinksToScan );
			#push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );	

		} elsif ( $ContentType =~ /bittorrent/i  ) { # wenn $CurrentUrlToScan torrent datei ist
			
			# CATE=en#DESC=Read or Die - OVA (GermanDub)#LINK=http://www.mininova.org/tor/471390
			
			# für alle einträge in $LinkScannedFile
			# my ( $LinkDesc, $ParentLink1, $LinkName ) = split(' ### ', $_);
			# if $ParentLink1 eq $ParentLink -> nimm die description

			open(TORRENT,">>$TorrentStoreFlatFile") or die;
				TORRENT->autoflush(1);
				TORRENT->blocking(0);
				flock(TORRENT, 2);
				print TORRENT "LANG=$CatLanguage#PARENT=$ParentLink#LINK=$CurrentUrlToScan\n";
			close TORRENT;

			#	print "#################################################\n";
			#	print "######## found on parent:		$ParentLink \n";
			#	print "######## desc:					$CurrentLinkDesc \n";
			#	print "######## LINK					$CurrentUrlToScan\n";
			#	print "#################################################\n";

			print "##### Torrent found: $CurrentUrlToScan ###### \n";

		}; # ENDE: Content type

	}; # if ( $HeadHashResultRef->{ 'STA' } == 1 ) {


	# speichere gescannte urls
	open(SCANNED,">>$LinkScannedFile") or die;
		SCANNED->autoflush(1);
		SCANNED->blocking(0);
		flock(SCANNED, 2);
		print SCANNED "$CurrentUrlToScan\n";
	close SCANNED;


	# return ($ParentUrl,$ParentDesc);
	return 1;

}; # sub Scanner(){