#!/usr/bin/perl -Imodules

#########################################
##### Author:		Sebastian Enger / B.Sc
##### CopyRight:	Sebastian Enger
##### LastModified	16.07.2006
##### Function:		
##### Todo:			
########################################

#############
###### run from path: /root/zoozle/indexer/GenerellSpider
#############

# perl -MCPAN -e 'force install "XML::TreePP"'
# perl -MCPAN -e 'force install "APR::URI"'

system("clear");


# lade immer alle module -> error checking

use strict;
use IO::Handle;
use Data::Dumper;
use TorrentIndexer::HTTP::Get;
use TorrentIndexer::HTTP::Head;
use TorrentIndexer::LINK::Extractor;
use TorrentIndexer::CHECKSUM::CheckSum;

use TorrentIndexer::TIME::Time;
use TorrentIndexer::HTML::HtmlParser;
use TorrentIndexer::CRYPTO::CryptoLibrary;
use TorrentIndexer::CRYPTO::SecureHashGenerator;
use TorrentIndexer::HTTP::RobotRules;
use TorrentIndexer::LINK::LinkDept;
use TorrentIndexer::LINK::IsSameLink;
use TorrentIndexer::BITTORRENT::Bittorrent;


# my $MainUrlToScan		= "http://www.zoozle.net/zoozle.php?q=test&s=bittorrent&l=de";
# my $MainUrlToScan		= "http://www.zoozle.net/links.html"; 

# my $MainUrlToScan		= "http://eselfilme.com/parents/home.htm";
# my $MainUrlToScan		= "http://fenopy.com/rss.xml?row=10&cat=5&type=2";
# my $MainUrlToScan		= "http://www.p2p-blog.de/feed/";
# my $MainUrlToScan		= "http://torrent.to/torrent/upload/serien/713/150713/1308-www.torrent.to...CSI.Miami.S04E08.Wunschmoerder.German.Dubbed.DVDRiP.WS.XviD-TvR.torrent";

# my $MainUrlToScan		= "http://www.freesexypics.de.vu"; 
# my $MainUrlToScan		= "http://www.zoozle.net/zoozle/information.html";



my %AlreadyScannedLinks	= ();
my $LinkInformation		= {};
# my $LinkHashRef		= {};
my @LinksToScan			= ();
my $TorrentStoreFlatFile;

my $CachePath			= "/root/zoozle/indexer/torrentindexer/cache"; 
mkdir $CachePath;						# later check if dir exits if not create, siehe client.pm easyspider
my $LinkScannedFile		= "/root/zoozle/indexer/torrentindexer/cache/linkscanned.txt"; 
unlink $LinkScannedFile;

my $ScanUntilDept		= 2;
my $CurrentDept		    = 0;

my $ScanDeptType		= 0;	# 0=linkdept | 1=diredept
my $StayOnSameHost		= 1;
my $AlreadyScannedFlag	= 0;


#######

$LinkInformation->{'LINKNAME'}			= undef;
$LinkInformation->{'LINKTITLE'}			= undef;
$LinkInformation->{'SCANNED'}			= undef;
$LinkInformation->{'PAGETITLE'}			= undef;
$LinkInformation->{'METADESC'}			= undef;
$LinkInformation->{'METAKEYS'}			= undef;
$LinkInformation->{'HASHID'}			= undef;
$LinkInformation->{'CLD'}				= undef;
$LinkInformation->{'PARENTURL'}			= undef;
$LinkInformation->{'PARENTDESC'}		= undef;


###########
### objekte initialisieren
###########


my $HEAD		= TorrentIndexer::HTTP::Head->new();
my $GET			= TorrentIndexer::HTTP::Get->new();
my $LINK		= TorrentIndexer::LINK::Extractor->new();
my $SAME		= TorrentIndexer::LINK::IsSameLink->new();
my $BITTORRENT	= TorrentIndexer::BITTORRENT::Bittorrent->new();


#########
### MAIN
#########

my $ParentUrl1 = "http://torrent.to/torrent/popup.php?Active=xxx&ID=151796";
my $ParentDesc1 = "TestDescription";

my (undef,undef,$Hostname,undef) = split('/', $ParentUrl1);
$TorrentStoreFlatFile	= "$CachePath/$Hostname-torrent.txt";


&Scanner($ParentUrl1, $ParentUrl1, $ParentDesc1);


	foreach my $HashRef ( @LinksToScan ){
		
		my $LinkName = $HashRef->{'LINKNAME'};
		# print "LINK FOUND: $LinkName \n";
		# next;

		open(SCANNED,"<$LinkScannedFile") or die;
			
			SCANNED->autoflush(1);
			SCANNED->blocking(0);
			flock(SCANNED, 2);
			
			while(<SCANNED>){
				# print "'$_' und '$LinkName'\n"; sleep 2;
				if ( lc($_) eq lc($LinkName) ) {
					$AlreadyScannedFlag = 1;
				}; # if 
			}; # while(<SCANNED>){

		close SCANNED;
	
		if ( $AlreadyScannedFlag == 0 ) {
			
			# wenn wir auf dem MainHost bleiben sollen
			if ( $StayOnSameHost == 1 ) {
			
				# print "VERLGEICHE: $ParentUrl, $LinkName\n";
				# vergleiche start url mit aktueller url aus $HashRef->{'LINKNAME'}
				my $IsSameHostName = $SAME->IsSameLink($ParentUrl1, $LinkName);
								
				if ( $IsSameHostName == 1 ) {
					
					# print "Now i would scan: $LinkName \n";
					# print "[$IsSameHostName] VERLGEICHE: $ParentUrl, $LinkName\n";
					# sleep 1;
					&Scanner( $ParentUrl1, $LinkName, $ParentDesc1 );	# parent, url_to_scan
									
				} else {
					
					# wenn der host nicht gleich der mainurl ist, dann markiere den link als gescanned
					# $AlreadyScannedLinks{$LinkName}++;

				}; # if ( $IsSameHostName == 1 ) {

			} else { 
				
				# wenn wir nicht auf dem mainurl bleiben sollen, sondern wandern dürfen
				#  &scanne $HashRef->{'LINKNAME'}
				# $AlreadyScannedLinks{$LinkName}++;

			}; #ENDE: if ( $StayOnSameHost == 1 ) {

		} else { # if ( $AlreadyScannedFlag == 1 ) {

			# print "ALREADY SCANNED :[COUNT] " . $AlreadyScannedLinks{$LinkName} . " for LINK $LinkName \n";
			# print "\t ALREADY SCANNED $LinkName \n";
			# sleep 1;

		}; # if ( exists($AlreadySc

		# $AlreadyScannedFlag wieder auf Null setzen
		$AlreadyScannedFlag = 0;

	}; # foreach my $HashRef ( @LinksToScan ){



sub Scanner(){

	my $ParentUrl			= shift;
	my $CurrentUrlToScan	= shift;
	my $ParentDesc			= shift;			

	print "SCANNE: '$CurrentUrlToScan'\n";
#	print "\t and i came from $ParentUrl\n";

	my $HeadHashResultRef	= $HEAD->Head($CurrentUrlToScan);
	my $HeadStatusCode		= $HeadHashResultRef->{ 'STA' };
	
	# print Dumper($HeadHashResultRef);
	# print "\n";

	if ( $HeadStatusCode == 1 ) {

		my $ContentType			= $HeadHashResultRef->{ 'TYP' };
		my $GetHashResultRef	= $GET->Get($CurrentUrlToScan);
		
		my $ResponseObj			= $GetHashResultRef->{ 'OBJ' };
		my $ResponseContent		= $GetHashResultRef->{ 'CNT' };
		my $CurrentLinkDept		= 0;

		# print "ContentTyp=$ContentType\n";
		# select(undef,undef,undef,0.1);

		if ( $ContentType =~ /html/i ) {	# wenn $CurrentUrlToScan html datei ist

			my $ScanLinkConfig = {
				"OBJ" => $ResponseObj,
				"CON" => $ResponseContent,
				"CDP" => $CurrentDept,
				"SUD" => $ScanUntilDept,
				"URL" => $ParentUrl,
				"PDC" => my $ParentDescription,
				"SDT" => $ScanDeptType,
			};

			my $LinkHashRef = $LINK->HtmlLinkExtractor( $ScanLinkConfig );
			push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );		
		
		} elsif ( $ContentType =~ /xml/i ) { # wenn $CurrentUrlToScan xml datei ist
			
			my $ScanLinkConfig = {
				"OBJ" => undef,
				"CON" => $ResponseContent,
				"CLD" => undef,
				"SUD" => undef,
				"URL" => $ParentUrl,
				"PDC" => undef,
				"SDT" => undef,
			};

			my $LinkHashRef = $LINK->XmlLinkExtractor( $ScanLinkConfig );
			push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );	

		} elsif ( $ContentType =~ /plain/i ) {  # wenn $CurrentUrlToScan text datei ist

			my $ScanLinkConfig = {
				"OBJ" => undef,
				"CON" => $ResponseContent,
				"CDP" => $CurrentDept,
				"SUD" => $ScanUntilDept,
				"URL" => $ParentUrl,
				"PDC" => undef,
				"SDT" => undef,
			};

			my $LinkHashRef = $LINK->TextLinkExtractor( $ScanLinkConfig );
			push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );	

		} elsif ( $ContentType =~ /bittorrent/i  ) { # wenn $CurrentUrlToScan torrent datei ist
			
			# ... dann finden wir auf $ParentUrl ( der url, in deren content wir dieses torrent file $CurrentUrlToScan gefunden haben) ... 
			# alle wichtigen informationen über das torrent
			
			#	my $CHECK			= TorrentIndexer::CHECKSUM::CheckSum->new();
			#	my $TorrentMD5		= $CHECK->MD5ToHEX( $ResponseContent );
			#	my $torrentFile		= "$CachePath/$TorrentMD5";
			#
			#	open(TORRENT,">$torrentFile") or die;
			#		binmode(TORRENT);
			#		print TORRENT $ResponseContent;
			#	close TORRENT or die;
			#
			#	my $BittorrentInformation = $BITTORRENT->getBittorrentInfo( $torrentFile );
			
			$TorrentStoreFlatFile

			print "#################################################\n";
			print "######## found on: $ParentUrl \n";
			print "######## desc: $ParentDesc \n";
			print "#################################################\n";
			
			exit;
					
		}; # ENDE: Content type

	}; # if ( $HeadHashResultRef->{ 'STA' } == 1 ) {


	# speichere gescannte urls
	open(SCANNED,">>$LinkScannedFile") or die;
		SCANNED->autoflush(1);
		SCANNED->blocking(0);
		flock(SCANNED, 2);
		print SCANNED "$CurrentUrlToScan";
	close SCANNED;


}; # sub Scanner(){