#!/usr/bin/perl -Imodules

#########################################
##### Author:		Sebastian Enger / B.Sc
##### CopyRight:	Sebastian Enger
##### LastModified	16.07.2006
##### Function:		
##### Todo:			
########################################

#############
###### run from path: /root/zoozle/indexer/GenerellSpider
#############

# perl -MCPAN -e 'force install "XML::TreePP"'
# perl -MCPAN -e 'force install "APR::URI"'

system("clear");


# lade immer alle module -> error checking

use strict;
use IO::Handle;
use Data::Dumper;
use TorrentIndexer::HTTP::Get;
use TorrentIndexer::HTTP::Head;
use TorrentIndexer::LINK::Extractor;
use TorrentIndexer::CHECKSUM::CheckSum;

use TorrentIndexer::TIME::Time;
use TorrentIndexer::HTML::HtmlParser;
use TorrentIndexer::CRYPTO::CryptoLibrary;
use TorrentIndexer::CRYPTO::SecureHashGenerator;
use TorrentIndexer::HTTP::RobotRules;
use TorrentIndexer::LINK::LinkDept;
use TorrentIndexer::LINK::IsSameLink;
use TorrentIndexer::BITTORRENT::Bittorrent;


# my $MainUrlToScan		= "http://www.zoozle.net/zoozle.php?q=test&s=bittorrent&l=de";
# my $MainUrlToScan		= "http://www.zoozle.net/links.html"; 

# my $MainUrlToScan		= "http://eselfilme.com/parents/home.htm";
# my $MainUrlToScan		= "http://fenopy.com/rss.xml?row=10&cat=5&type=2";
# my $MainUrlToScan		= "http://www.p2p-blog.de/feed/";
# my $MainUrlToScan		= "http://torrent.to/torrent/upload/serien/713/150713/1308-www.torrent.to...CSI.Miami.S04E08.Wunschmoerder.German.Dubbed.DVDRiP.WS.XviD-TvR.torrent";

# my $MainUrlToScan		= "http://www.freesexypics.de.vu"; 
# my $MainUrlToScan		= "http://www.zoozle.net/zoozle/information.html";



my %AlreadyScannedLinks	= ();
my $LinkInformation		= {};
# my $LinkHashRef		= {};
my @LinksToScan			= ();

my $CachePath			= "./cache"; mkdir $CachePath;	# later check if dir exits if not create, siehe client.pm easyspider
my $LinkScannedFile		= "./cache/linkscanned.txt";

my $ScanUntilDept		= 2;
my $CurrentDept		    = 0;

my $ScanDeptType		= 0;	# 0=linkdept | 1=diredept
my $StayOnSameHost		= 1;



#######

	$LinkInformation->{'LINKNAME'}			= undef;
	$LinkInformation->{'LINKTITLE'}			= undef;
	$LinkInformation->{'SCANNED'}			= undef;
	$LinkInformation->{'PAGETITLE'}			= undef;
	$LinkInformation->{'METADESC'}			= undef;
	$LinkInformation->{'METAKEYS'}			= undef;
	$LinkInformation->{'HASHID'}			= undef;
	$LinkInformation->{'CLD'}				= undef;
	$LinkInformation->{'PARENTURL'}			= undef;
	$LinkInformation->{'PARENTDESC'}		= undef;


###########
### objekte initialisieren
###########


my $HEAD		= TorrentIndexer::HTTP::Head->new();
my $GET			= TorrentIndexer::HTTP::Get->new();
my $LINK		= TorrentIndexer::LINK::Extractor->new();
my $SAME		= TorrentIndexer::LINK::IsSameLink->new();
my $BITTORRENT	= TorrentIndexer::BITTORRENT::Bittorrent->new();


#########
### MAIN
#########

my $ParentUrl = "http://torrent.to/torrent/popup.php?Active=doku&ID=151169";

my $LINKS = &Scanner("$ParentUrl", "$ParentUrl", "HASHID", 'LINKHASHREF');

	foreach my $HashRef ( @{$LINKS} ){
		
		my $LinkName = $HashRef->{'LINKNAME'};
		# print "LINK FOUND: $LinkName \n";
		# next;

		$AlreadyScannedLinks{$LinkName}++;
	

		if ( $AlreadyScannedLinks{$LinkName} >= 2 ) {
			
			# print "ALREADY SCANNED :[COUNT] " . $AlreadyScannedLinks{$LinkName} . " for LINK $LinkName \n";
			# print "\t ALREADY SCANNED $LinkName \n";
			# sleep 1;

		} else {

			# wenn wir auf dem MainHost bleiben sollen
			if ( $StayOnSameHost == 1 ) {
			
				# print "VERLGEICHE: $ParentUrl, $LinkName\n";
				# vergleiche start url mit aktueller url aus $HashRef->{'LINKNAME'}
				my $IsSameHostName = $SAME->IsSameLink($ParentUrl, $LinkName);
				# print "[$IsSameHostName] VERLGEICHE: $ParentUrl, $LinkName\n";
				
				if ( $IsSameHostName == 1 ) {
					
					# print "Now i would scan: $LinkName \n";
					# print "[$IsSameHostName] VERLGEICHE: $ParentUrl, $LinkName\n";
					# sleep 1;
					&Scanner( $ParentUrl, $LinkName, $HashRef->{'HASHID'}, $LINKS );	# parent, url_to_scan
					# $AlreadyScannedLinks{$LinkName}++;
				
				} else {
					
					# wenn der host nicht gleich der mainurl ist, dann markiere den link als gescanned
					# $AlreadyScannedLinks{$LinkName}++;

				}; # if ( $IsSameHostName == 1 ) {

			} else { 
				
				# wenn wir nicht auf dem mainurl bleiben sollen, sondern wandern dürfen
				#  &scanne $HashRef->{'LINKNAME'}
				# $AlreadyScannedLinks{$LinkName}++;

			}; #ENDE: if ( $StayOnSameHost == 1 ) {

		}; # if ( exists($AlreadySc

	}; # foreach my $HashRef ( @LinksToScan ){





sub Scanner(){

	my $ParentUrl			= shift;
	my $CurrentUrlToScan	= shift;
	my $CurrentHashId		= shift;
	my $LINKSHASHREF		= shift;			

	print "SCANNE: $CurrentUrlToScan\n";
#	print "\t and i came from $ParentUrl\n";
	
	# todo: speicher gescannte links in einem array, das global gültig ist


	my $HeadHashResultRef	= $HEAD->Head($CurrentUrlToScan);
	my $HeadStatusCode		= $HeadHashResultRef->{ 'STA' };
	
	# print Dumper($HeadHashResultRef);
	# print "\n";

	if ( $HeadStatusCode == 1 ) {

		my $ContentType			= $HeadHashResultRef->{ 'TYP' };
		my $GetHashResultRef	= $GET->Get($CurrentUrlToScan);
		
		my $ResponseObj			= $GetHashResultRef->{ 'OBJ' };
		my $ResponseContent		= $GetHashResultRef->{ 'CNT' };
		my $CurrentLinkDept		= 0;

	#	print "ContentTyp=" . $HeadHashResultRef->{ 'TYP' } . "\n";
		print "ContentTyp=$ContentType\n";
		select(undef,undef,undef,0.1);

		if ( $ContentType =~ /html/i ) {	# wenn $CurrentUrlToScan html datei ist

			my $ScanLinkConfig = {
				"OBJ" => $ResponseObj,
				"CON" => $ResponseContent,
				"CDP" => $CurrentDept,
				"SUD" => $ScanUntilDept,
				"URL" => $ParentUrl,
				"PDC" => my $ParentDescription,
				"SDT" => $ScanDeptType,
			};

			my $LinkHashRef = $LINK->HtmlLinkExtractor( $ScanLinkConfig );
			push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );		
		

		} elsif ( $ContentType =~ /xml/i ) { # wenn $CurrentUrlToScan xml datei ist
			
			my $ScanLinkConfig = {
				"OBJ" => undef,
				"CON" => $ResponseContent,
				"CLD" => undef,
				"SUD" => undef,
				"URL" => $ParentUrl,
				"PDC" => undef,
				"SDT" => undef,
			};

			my $LinkHashRef = $LINK->XmlLinkExtractor( $ScanLinkConfig );
			push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );	

		} elsif ( $ContentType =~ /plain/i ) {  # wenn $CurrentUrlToScan text datei ist

			my $ScanLinkConfig = {
				"OBJ" => undef,
				"CON" => $ResponseContent,
				"CDP" => $CurrentDept,
				"SUD" => $ScanUntilDept,
				"URL" => $ParentUrl,
				"PDC" => undef,
				"SDT" => undef,
			};

			my $LinkHashRef = $LINK->TextLinkExtractor( $ScanLinkConfig );
			push (@LinksToScan, @{$LinkHashRef->{'LINKS'}} );	

		} elsif ( $ContentType =~ /bittorrent/i  ) { # wenn $CurrentUrlToScan torrent datei ist
			
			# ... dann finden wir auf $ParentUrl ( der url, in deren content wir dieses torrent file $CurrentUrlToScan gefunden haben) ... 
			# alle wichtigen informationen über das torrent
			
			my $BittorrentInformation = $BITTORRENT->getBittorrentInfo($CachePath, $ResponseContent, );
			
			foreach my $HashRef ( @{$LINKSHASHREF->{'LINKS'}} ){
				
				my $LinkName	= $HashRef->{'LINKNAME'};
				my $HashID		= $HashRef->{'HASHID'};
				my $ParentURL	= $HashRef->{'PARENTURL'};
				my $ParentDESC	= $HashRef->{'PARENTDESC'};
			
				if ( $CurrentHashId eq $HashID ) {

					print "#################################################\n";
					print "######## torrent found: " . $BittorrentInformation->{'FILENAME'} . " \n";
					print "######## found on: $ParentURL \n";
					print "######## desc: $ParentDESC \n";
					print "#################################################\n";

				}; # if ( $CurrentHashId eq $HashID ) {

			}; # foreach my $HashRef ( @{$LinkHashRef->{'LINKS'}} ){
		
		}; # ENDE: Content type

	}; # if ( $HeadHashResultRef->{ 'STA' } == 1 ) {


	# speichere gescannte urls
	open(SCANNED,">$LinkScannedFile") or die;
		SCANNED->autoflush(1);
		SCANNED->blocking(0);
		flock(SCANNED, 2);
		print SCANNED "$CurrentUrlToScan\n";
	close SCANNED;

	
	return \@LinksToScan;

}; # sub Scanner(){