#!/usr/bin/perl

use LWP::UserAgent;
use HTTP::Cookies;
use HTTP::Request;
use Tie::File;

$PROXYTMP	= "proxy.tmp.txt";
$PROXYGO	= "proxy.working.txt";
$TESTURL	= "http://www.zoozle.net/zoozle/information.html";
$UA			= "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; YPC 3.0.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";

############# Subroutine zum erstellen eines HTTP Requestes:Start
# Aufgabe: &http_get - erstelle und sende http request an übergebene url
# Rückgabe: content der angefragten uri/url

sub http_get() {
	
	my ($url, $timeout, $useragent, $proxy) = @_;
	my $UA = LWP::UserAgent->new( keep_alive => 1 );

	$UA->agent($useragent);
	$UA->timeout($timeout);
	$UA->max_size(950_000);
	#$UA->proxy(["http"], "http://$proxy") if ($proxy ne '');
  
	my $jar_jar = HTTP::Cookies->new
		(file => ".Cookies" || ".Cookies.txt",
			autosave => 1,
			max_cookie_size => 40960,
			max_cookies_per_domain => 10000, );
			$UA->cookie_jar($jar_jar );
		
	my $req = HTTP::Request->new(GET => $url);
	
	# Pass request to the user agent and get a response back
  	
	$req->referer($url);
	
	my $res = $UA->request($req);

	if ($res->is_success) {
		return $res->content;
  	} else {
      	#return "<html><body> Spider::Easyspider::http_get(): " . $res->status_line . "</body></html>";
		return 0;
  	};

}; # sub http_get {}
############# Subroutine zum erstellen eines HTTP Requestes:Ende


sub giveWorkingProxys(){
	
#	my $proxyServerRessource = {
#		"1" => "http://www.samair.ru/proxy/",
#		"2" => "http://www.samair.ru/proxy/proxy-02.htm",
#		"3" => "http://www.samair.ru/proxy/proxy-03.htm",
#		"4" => "http://www.samair.ru/proxy/proxy-04.htm",
##		"5" => "http://www.samair.ru/proxy/proxy-05.htm",
#		"6" => "http://www.samair.ru/proxy/proxy-06.htm",
#		"7" => "http://www.samair.ru/proxy/proxy-07.htm",
#		"8" => "http://www.samair.ru/proxy/proxy-08.htm",
#		"9" => "http://www.samair.ru/proxy/proxy-09.htm",
#		"10" => "http://www.samair.ru/proxy/proxy-10.htm",
#		"11" => "http://www.samair.ru/proxy/proxy-11.htm",
#		"12" => "http://www.samair.ru/proxy/proxy-12.htm",
#		"13" => "http://www.samair.ru/proxy/proxy-13.htm",
#		"14" => "http://www.samair.ru/proxy/proxy-14.htm",
#		"15" => "http://www.samair.ru/proxy/proxy-15.htm",
#		"16" => "http://www.samair.ru/proxy/proxy-16.htm",
#		"17" => "http://www.samair.ru/proxy/proxy-17.htm",
#		"18" => "http://www.samair.ru/proxy/proxy-18.htm",
#		"19" => "http://www.samair.ru/proxy/proxy-19.htm",
#		"20" => "http://www.samair.ru/proxy/proxy-20.htm"
#	};

#	tie @PROXY, 'Tie::File', $PROXYTMP or die;
#	while( my(undef, $url) = each(%{$proxyServerRessource})){
#		foreach $line ( split('<pre>', &http_get("http://www.samair.ru/proxy/proxy-13.htm")) ) {
#			
#			print "$line\n";
#			my ($IP,$PORT)	= split(':', $line);
#			if( ( $IP =~ /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/ || $1 > 255 || $2 > 255 || $3 > 255 || $4 > 255 ) && ($PORT =~ /(\d{1,5})/)){
#				push(@PROXY, "$IP:$PORT");
#				print "$IP:$PORT\n";
#			} else {
#				print "$IP:$PORT\n";
#			};
#		};
#	};
#	untie @PROXY;
#	$pcount = "0";
#	

	tie @PROXY, 'Tie::File', $PROXYTMP or die;
		push(@PROXYS, @PROXY);
	untie @PROXY;
	
	tie @PROXY, 'Tie::File', $PROXYGO or die;

#PROXYCHECK:	
#	foreach $proxy (@PROXYS) {
#		
#		if ( &http_get($TESTURL, "30", $UA, $proxy) == 0 ){
#			print "";
#			next PROXYCHECK;		
#		} else {
#			push(@PROXYGO, "$proxy\n");
#			print "[$pcount] WORKING->$proxy \n";
#		};
#		$pcount++;
#	};

	foreach $line (@PROXYS) {
		my ($imp,undef,undef) = split('\s', $line);
		push(@FINAL, $imp);
	}

	untie @PROXYGO;
#
	return \@FINAL;		# referenz auf proxys zurückgeben

};


############# Subroutine zur Extrahierung von Links: Start
# Aufgabe: &get_links - extrahiere links aus übergebenen html datenstrom 
# Rückgabe: @links mit Links aus übergebenen html datenstrom

sub get_links() {
	
	my ($page, $starturl) = @_;
	my @content = split('a href', $page);
	my $frame = "0";
	
	chomp($starturl);
	
	foreach $line (@content){
		my ($tmp,$link,undef) = split('"', $line);
		next if ($link =~ /mailto/);
		
		# wenn es sich bei einer url um eine relative url handelt, wandle es mit make_absolute_url um!	
		if ($link !~ /^http/){
			next if ($tmp =~ /</i);
			my $abs_link = &make_absolute_url($starturl, $link);
			push(@links, $abs_link);
		} else {
			push(@links, $link);
		};
	};

	undef @content;
	@links = grep { ! $seen{$_} ++ } @links;
	
	if (defined(@links)){
		
		return @links;
		
	} else {
		
		my @content2 = split('\'', $page);
		my @frame_content = split('<', $page);
	
		foreach $frame_line (@frame_content){
			($tmp,$link,undef) = split('"', $frame_line);
			if ($tmp =~ /FRAME SRC/i){
				push(@links2, $link);
				$frame = "1";
			};
		};
		
		undef @frame_content;
		
		if ($frame == "0") {
		
			foreach $line (@content2){
		
				my ($tmp,$link,undef) = split('"', $line);
				next if ($link =~ /mailto/);
		
				if ($link !~ /^http/){
					next if ($tmp =~ /</i);
					my $abs_link = &make_absolute_url($starturl, $link);
					push(@links2, $abs_link);
				} else {
					push(@links2, $link);
				};
			};
			
		};
		
		undef @content2, $frame;
		@links2 = grep { ! $seen2{$_} ++ } @links2;
		return @links2;
	};

}; # sub get_links {}
############# Subroutine zur Extrahierung von Links: Ende   


################### Subroutine zum Umwandeln von relativen in absolute Urls: Start
# Aufgabe: &make_absolute_url() - relativ->absolute urls umwandeln
# Rückgabe: umgewandelte Url

sub make_absolute_url() {

 	my ($parent_url, $child_url) = (shift,shift);
		
	my $parent_bak = $parent_url;
		
	if ($child_url =~/^#/){
		return undef;
	};
	
	my $hack;
	if ($child_url =~ m|^/|) {
		$parent_url =~ s|^(http://[\w.]+)?/.*$|$1|i;
		return $parent_url.$child_url;
	};

	if ($child_url =~ m|^\.\.\/|i){
		$parent_url =~ s/\/[^\/|^~]+$//; # Strip filename (fix: DL)
		if ($parent_url =~ /\/$/){$parent_url =~ s/\/$//;}	# (DL)
		if ($child_url =~ /^\.\//){$child_url =~ s/^\.\///;}	# (DL)
		while ($child_url=~s/^\.\.\///gs ){
			$parent_url =~s/[^\/]+\/?$//;
		}
		$child_url = $parent_url.$child_url;

	} elsif ($child_url !~ m/^http:\/\//i){
		
		# Assume relative path needs dir
		$parent_url =~ s/\/[^\/]+$//;	# Strip filename
		if ($parent_url =~ /\/$/){ chop $parent_url }
		$child_url = $parent_url .'/'.$child_url;
	}; 
	
	my $child_bak = $child_url;
	substr($child_bak, 0,6) = "";
	
	if ( $child_bak =~ /^\// ){
		#	print "FALSE: $child_bak\n";

	} else {
		#print "RIGHT: $child_bak\n";
		$child_url = $parent_bak .'/'. $child_bak;
	};
	
	return $child_url;

}# sub make_absolute_url {}
################### Subroutine zum Umwandeln von relativen in absolute Urls: Ende


sub deleteSpecialChars() {

	my $del_badchar = shift;
	
		$del_badchar =~ s/&nbsp//g;
		$del_badchar =~ s/nbsp//g;
		$del_badchar =~ s/\`/ /g;
		$del_badchar =~ s/\'/ /g;
		$del_badchar =~ s/\?/+/g;
		$del_badchar =~ s/%//g;
		$del_badchar =~ s/$//g;
		$del_badchar =~ s/§//g;
		$del_badchar =~ s/!//g;
		$del_badchar =~ s/&//g;
		$del_badchar =~ s/\{//g;
		$del_badchar =~ s/\}//g;
		$del_badchar =~ s/\(//g;
		$del_badchar =~ s/\)//g;
		$del_badchar =~ s/\[//g;
		$del_badchar =~ s/\]//g;
		$del_badchar =~ s/\=//g;
		$del_badchar =~ s/\\//g;
		$del_badchar =~ s/\#//g;
		$del_badchar =~ s/\,//g;
		$del_badchar =~ s/\;//g;
		$del_badchar =~ s/\|//g;
		$del_badchar =~ s/\</ /g;
		$del_badchar =~ s/\>/ /g;
		$del_badchar =~ s/\///g;
		$del_badchar =~ s/°//g;
		$del_badchar =~ s/\^//g;
		$del_badchar =~ s/ß/ss/g;
		$del_badchar =~ s/\|//g;
		$del_badchar =~ s/=&szlig;=/ss/g;
		$del_badchar =~ s/\./ /g;
		$del_badchar =~ s/\-/ /g;
		$del_badchar =~ s/ g / /g;
		$del_badchar =~ s/nbsp/ /g;
		$del_badchar =~ s/\+/ /g;
		$del_badchar =~ s/\*/ /g;
		$del_badchar =~ s/\&+\#+(\d)+\;/ /g;

	return ($del_badchar);
	
} # sub deleteSpecialChars() {}


1;