easyspider - Version1 - contrib - SOURCE CODE

#!/usr/bin/perl -Imodules

use Spider::Parser;

my $PARSER = Spider::Parser->new();


open(RH,"$ARGV[0]");
@content = <RH>;
close RH;


my $html_content = $PARSER->HtmlParser(@content);
print $html_content;

#!/usr/bin/perl

$file = $ARGV[0];

open(RH,"<$file");
	 # slurp everythin into one scalar
	 # my $content1 = do { local( $/ ) ; <RH> } ;
my @content = <RH>;
close RH;

$content = &slurp_from_array_into_scalar(@content);

$content =~ s/<!--(.*?)-->//igms;					# remove comments
$content =~ s/<script.*?>(.*?)<\/script>//igsm;		# remove scripts
$content =~ s/<style.*?>(.*?)<\/style>//igsm;		# remove style
$content =~ s/<.*?>//igms;							# remove html tags
$content = &html_to_ascii_parser($content);

foreach $line ( split(//, $content) ){
	$c++;
	if ($c >= "120") {
		$line =~ s/$line/$line\n/;
		$c = "0";
	}
	print $line;
	
}

sub slurp_from_array_into_scalar {
	
	foreach $content (@_) {
		$html_one_line = $html_one_line . $content;
	}

	return $html_one_line;
	
}

sub html_to_ascii_parser() {

	my $HTML = @_[0];
	my(@HTMLPage, $SymbLine, $ascii, $html);

	$HTML =~ s/&nbsp;/ /g;
	$HTML =~ s/\s\s*/ /g;
	$HTML =~ s/\n\s*\n\s*/\n\n/g;
	$HTML =~ s/\n */\n/g;
	
	foreach $SymbLine (&HTMLSymb) {
		($ascii, $html) = split(/\s\s*/,$SymbLine);

... [truncated, 121 more lines] ...

# /usr/bin/perl -Imodules
# file: web_serial.pl
# Figure 14.2: The baseline server handles requests serially

use strict;
use IO::Socket;
use Web;

my $port = shift || 80;
my $socket = IO::Socket::INET->new( LocalPort => $port,
				    Listen    => SOMAXCONN,
				    Reuse     => 1 ) or die "Can't create listen socket: $!";
while (my $c = $socket->accept) {
  
  handle_connection($c);
  close $c;
};
close $socket;

# /usr/bin/perl -Imodules
# file: web_serial.pl
# Figure 14.2: The baseline server handles requests serially

use IO::Socket;
use Web;

my $port = shift || 80;
my $socket = IO::Socket::INET->new( LocalPort => $port,
				    Listen    => SOMAXCONN,
				    Reuse     => 1 ) or die "Can't create listen socket: $!";

my $pid;

print "#####################################\n";
print "-= # WebServer started on Port 80 #=-\n";
print "#####################################\n";

#while (my $c = $socket->accept) {
#  
#  handle_connection($c);
#  close $c;
#};
#close $socket;

while (my $c = $socket->accept) {
	
	$socket->autoflush(0);
	$ClientSocket_ip = getpeername($c);
	($port, $ipaddr) = unpack_sockaddr_in($ClientSocket_ip);
	$ClientSocket_ip = inet_ntoa($ipaddr);

	if ($pid = fork) {
		close $socket;
	} elsif (defined $pid) {
		print "\t -=# Serving '$ClientSocket_ip' #=- \n";
		handle_connection($c);
		close $c;
		undef $pid;
	} else {
		die "fork error: $!";
	};
};

#!/usr/bin/perl

$file = $ARGV[0];

@valid = (" ","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z",".","!","?","\"","%","&","/","(",")","=","?","`","","","",",",".","-","+","*",",",";","+","-",">","<","@","[","]","}","{","=","~","#");
$count = 1;

foreach (@valid) {
	ord($_);
}

open(RH, $file);
binmode(RH);

while (<RH>) {
	@con = split('',$_);
	foreach $con (@con) {
		ord($con);
		foreach $val (@valid) {
			if ($con eq $val){
				push(@1,$con);
			};
		};
	};
	@con = ();
};

open(WH,">test.out");
	print WH @1;
close WH;

open(RH,"<test.out");
	my $content = do { local( $/ ) ; <RH> } ;
	$content =~ s/\.\./\./ig;
	$content =~ s/\\/\/ig;
	$content =~ s/\-\-/\-/ig;
	$content =~ s/\+\+/\+/ig;
	$content =~ s/\,\,/\,/ig;
	$content =~ s/\;\;/\;/ig;
	$content =~ s/\=\=/\=/ig;
close RH;

$wanted = length($content)/5.7;
$wanted = int($wanted+.99999);

@content = split('',$content);

foreach $line (@content) {
	$count++;
	if ($count <= $wanted) {

... [truncated, 10 more lines] ...

#!/usr/bin/perl

$file = $ARGV[0];

@valid = (" ","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z",".","!","?","\"","%","&","/","(",")","=","?","`","","","",",",".","-","+","*",",",";","+","-",">","<","@","[","]","}","{","=","~","#");
$count = 1;

foreach (@valid) {
	ord($_);
}

open(RH, $file);
binmode(RH);

while (<RH>) {
	@con = split('',$_);
	foreach $con (@con) {
		ord($con);
		foreach $val (@valid) {
			if ($con eq $val){
				push(@1,$con);
			};
		};
	};
	@con = ();
};

open(WH,">test.out");
	print WH @1;
close WH;

open(RH,"<test.out");
	my $content = do { local( $/ ) ; <RH> } ;
	$content =~ s/\.\./\./ig;
	$content =~ s/\\/\/ig;
	$content =~ s/\-\-/\-/ig;
	$content =~ s/\+\+/\+/ig;
	$content =~ s/\,\,/\,/ig;
	$content =~ s/\;\;/\;/ig;
	$content =~ s/\=\=/\=/ig;
close RH;

$wanted = length($content)/5.7;
$wanted = int($wanted+.99999);

@content = split('',$content);

foreach $line (@content) {
	$count++;
	if ($count <= $wanted) {

... [truncated, 10 more lines] ...

#!/usr/bin/perl

use CGI qw(param);
use DBI;
use Crypt::Blowfish;

$db_user 	= "root";
$db_pass 	= "973fuckASI6";
$c_time 	= time();
$token 		= param("q");

print "Content-type: text/plain\n\n";

#SELECT valid_key,valid_until,is_valid FROM `authentification` WHERE valid_key = 'TESTKEY'
# insert into authentification (valid_key, valid_until,is_valid) values('TESTKEY1','1120896154','1');
my $dbh = DBI->connect('DBI:mysql:spider_authentification', $db_user, $db_pass) or die "failed to open $dbh: $!\n";
my $sth = $dbh->prepare("SELECT valid_key,valid_until,is_valid FROM `authentification` WHERE valid_key = '$token'");
$sth->execute();
  
while (my $ref = $sth->fetchrow_hashref()) {
   	$is_vaild 	= $ref->{'is_valid'};
   	$valid_until 	= $ref->{'valid_until'};
	$valid_key	= $ref->{'valid_key'};
}
$sth->finish();
$dbh->disconnect();

if ($token eq $valid_key){
	if ($c_time > $valid_until){
		#print "Key abgelaufen!\n";
		#print "$c_time und $valid_until";
		exit(0);
	} else {
		if ($is_vaild == "1"){
			#print "key noch valid";
			print "1";
		} else {
			#print "key nicht valid - breche ab";
			exit(0);
		}
	}
	
} else {
	#print "TOKEN mismatch";
	print "0";
	exit(0);
}

#!/usr/bin/perl

use Tie::File;
use LWP::Simple;

$url = $ARGV[0];
$depth = $ARGV[1];

#sub client_easyspider_mirror(){

my $config_path = "config";
mkdir $config_path;

$host = &get_hostname_from_url($url);

if ($depth == "1") {
	#scanne erstmal im ersten level nach allen links
	print "[$url]: mirroring with depth[1]\n";
	@links_1	= &get_links_from_url($url);
	
	foreach $link (@links_1){
		# skippe weiterleitungenen zu anderen http ressourcen, tool hngt sich dabei sonst auf
		my (undef, $zwei, undef) = split('http',$link);
		next if ($zwei =~ /(\w)/ig);

		print "#######################################\n";
		print "Mirror: [$link]\n";
		print "#######################################\n";
		&create_config_file($link, "$config_path/config.$host");
		&mirror_web_documents("$config_path/config.$host");
	}
	undef @links_1;

} elsif ($depth == "2") {

	print "[$url]: mirroring with depth[2]\n";
	#scanne erstmal im ersten level nach allen links
	my @links_1	= &get_links_from_url($url);

	foreach $link (@links_1){
		my (undef, $zwei, undef) = split('http',$link);
		next if ($zwei =~ /(\w)/ig);
				
		print "#######################################\n";
		print "Mirror: [$link]\n";
		print "#######################################\n";
		&create_config_file($link, "$config_path/config.$host");
		&mirror_web_documents("$config_path/config.$host");
	}


... [truncated, 2192 more lines] ...