zoozle-download-search-engine - 2010-zoozlebots - new-bots-X - contrib - SOURCE CODE

#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";

# renicing to 20 !
system("renice 20 $$");
system("clear");

require "/root/zoozle/new-bots-X/functions.pl";

# module laden
use strict;
no strict "refs";
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;


my $torrentflatfile		= "/root/zoozle/new-bots-X/emule_news.txt";
unlink $torrentflatfile;





... [truncated, 162 more lines] ...

#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";

# renicing to 20 !
system("renice 20 $$");
system("clear");

require "/root/zoozle/new-bots-X/functions.pl";

# module laden
use strict;
no strict "refs";
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;


my $torrentflatfile		= "/root/zoozle/new-bots-X/emule_news.txt";
unlink $torrentflatfile;





... [truncated, 209 more lines] ...

#!/usr/bin/perl

require "functions.pl";


while(1){

	my $esel = "/root/zoozle/updates/emule_news.txt";
	my $torr = "/root/zoozle/updates/torrent_news.txt";

	if ( -e $torr && -e $esel) {
		$done = 1;
	};


	InstallToDatabase($esel) if ( -e $esel );
	rename $esel => "/root/zoozle/updates/done/emule_news.txt";

	InstallToDatabase($torr) if ( -e $torr );
	rename $torr => "/root/zoozle/updates/done/torrent_news.txt";

	$now_string = localtime;
	print "Finished Installing at $now_string\n" if ($done == 1);

	$done = 0;
	sleep 3600;

};


sub InstallToDatabase(){

	my $file = shift;

	use DBI();

	$drh = DBI->install_driver("mysql");

	my $DBHOST		= "localhost";
	my $DBNAME		= "zoozle";
	my $DBUSER		= "zoozle";	
	my $DBPASS		= "zoozle23!99";	

	my $dbh = DBI->connect("DBI:mysql:database=$DBNAME;host=$DBHOST", "$DBUSER", "$DBPASS", {'RaiseError' => 0});

	&add_content_toSQLDB($file, $dbh);
	&add_content_toREALSQLDB($file, $dbh);

	$dbh->disconnect;


... [truncated, 120 more lines] ...

#!/usr/bin/perl

use LWP::UserAgent;
use HTTP::Cookies;
use HTTP::Request;
use Tie::File;

AddContentToSqlEMULE("/root/zoozle/new-bots-X/emule_news.txt");
AddContentToSqlTORRENT("/root/zoozle/new-bots-X/torrent_news.txt");
exit;


sub AddContentToSqlTORRENT() {

	my $filename	= shift;
	my $date		= &getDATE();

	use DBI();
	my $drh			= DBI->install_driver("mysql");

	my $DBHOST		= "localhost";
	my $DBNAME		= "zoozle";
	my $DBUSER		= "zoozle";	
	my $DBPASS		= "zoozle23!99";	

	my $dbh			= DBI->connect("DBI:mysql:database=$DBNAME;host=$DBHOST", "$DBUSER", "$DBPASS", {'RaiseError' => 0});
	my $count		= 0;

	open(RH1,"<$filename") or warn "$! - $filename \n";

		foreach (<RH1>) {
			
			my ( $cat, $desc, $link ) = split('#', $_);
			$link =~ s/LINK=//;
			$desc =~ s/DESC=//;
			$cat =~ s/CATE=//;
			
			next if ( $_ !~ /^CATE=/ );
			next if ( $_ =~ /Torrent does not exist/ig );
			next if ( $link =~ /megawerbung/);
			
			$cat	= &deleteSpecialChars($cat);
			$desc	= &deleteSpecialChars($desc);
			$count++;

			$desc =~ s/&+(\w)+;/ /i;
			$desc =~ s/&#+(\w)+;/ /i;
			$desc =~ s/&+(\d)+;/ /i;
			$desc =~ s/&#+(\d)+;/ /i;


... [truncated, 156 more lines] ...

#!/usr/bin/perl

use LWP::UserAgent;
use HTTP::Cookies;
use HTTP::Request;
use Tie::File;

#AddContentToSqlEMULE("/root/zoozle/new-bots-X/titanmule_all_news.txt");
AddContentToSqlEMULE("/root/zoozle/new-bots-X/titanmule_de_news.txt");
AddContentToSqlEMULE("/root/zoozle/new-bots-X/titanmule_en_news.txt");
exit;





sub AddContentToSqlEMULE() {

	my $filename	= shift;
	my $date		= &getDATE();

	use DBI();
	my $drh			= DBI->install_driver("mysql");

	my $DBHOST		= "localhost";
	my $DBNAME		= "zoozle";
	my $DBUSER		= "zoozle";	
	my $DBPASS		= "zoozle23!99";	

	my $dbh			= DBI->connect("DBI:mysql:database=$DBNAME;host=$DBHOST", "$DBUSER", "$DBPASS", {'RaiseError' => 0});
	my $count		= 0;

	open(RH1,"<$filename") or warn "$! - $filename \n";

		foreach (<RH1>) {
			
			my ( $cat, $desc, $link ) = split('#', $_);
			$link =~ s/LINK=//;
			$desc =~ s/DESC=//;
			$cat =~ s/CATE=//;
			
			next if ( $_ !~ /^CATE=/ );
			next if ( $_ =~ /Torrent does not exist/ig );
			next if ( $link =~ /megawerbung/);
			
			$cat	= &deleteSpecialChars($cat);
			$desc	= &deleteSpecialChars($desc);
			$count++;

			$desc =~ s/&+(\w)+;/ /i;

... [truncated, 160 more lines] ...

#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";
require "/root/zoozle/new-bots-X/functions.pl";

# renicing to 20 !
system("renice 20 $$");
system("clear");

# module laden
use strict;
no strict "refs";
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;


my $torrentflatfile		= "/root/zoozle/new-bots-X/torrent_news.txt";
unlink $torrentflatfile;

# filedonkey emule

my %LinksToScanDE = (		# 27 torrents
		"http://bitreactor.to/index"											=> '\/torrent\/',			# http://bitreactor.to/torrent/592734-filme-divx-xvid-otac-na-sluzbenom-putu-1985

... [truncated, 253 more lines] ...

#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";
require "/root/zoozle/new-bots-X/functions.pl";

# renicing to 20 !
system("renice 20 $$");
system("clear");

# module laden
use strict;
no strict "refs";
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;


my $torrentflatfile		= "/root/zoozle/new-bots-X/torrent_news.txt";
unlink $torrentflatfile;

# filedonkey emule

my %LinksToScanDE = (		# 27 torrents
		"http://bitreactor.to/index"											=> '\/torrent\/',			# http://bitreactor.to/torrent/592734-filme-divx-xvid-otac-na-sluzbenom-putu-1985

... [truncated, 295 more lines] ...

#!/usr/bin/perl

#use Net::SCP::Expect;
# perl -MCPAN -e 'install "Net::SCP::Expect"'
require "functions.pl";


CACHE();
sub CACHE(){

	my $timeout	= "300";
	my $ua		= "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.6) Gecko/20040206 Firefox/1.0.1";

	my %CACHING = (
		"/server/wwwroot/index.html" => "http://www.zoozle.org/index.php",		# .net: "/srv/server/wwwroot/index.html" => "http://www.zoozle.net/index.php",
	#	"/server/en.wwwroot/index.html" => "http://en.zoozle.net/index.php",
	#	"/srv/server/de.wwwroot/index.html" => "http://de.zoozle.net/index.php",

	#	"/server/apache2/htdocs/de/index.html" => "http://85.214.90.176/de/index_de.php",
	#	"/server/apache2/htdocs/org/index.html" => "http://85.214.90.176/org/index_org.php",
	#	"/server/apache2/htdocs/devu/index.html" => "http://85.214.90.176/devu/index_devu.php"
	);

	while( my ( $path, $url ) = each(%CACHING) ){
		my $page = &http_get($url, $timeout, $ua, );
		print "Generate Cache for $url\n";
		
		if ( length($page) >= 1000 ) {
		
			unlink $path;
			open(WH,">$path");
				print WH $page;
			close WH;

		}; # if ( length($page) >= 1000 ) {

	}; # while( my ( $path, $url ) = each(%CACHING) ){

	return 1;

}; # sub CACHE(){}

#!/usr/bin/perl

#use Net::SCP::Expect;
# perl -MCPAN -e 'install "Net::SCP::Expect"'
require "functions.pl";


while(1){

	system("perl /root/zoozle/new-bots-X/EmuleNewsCrawler.pl");
	system("perl /root/zoozle/new-bots-X/TorrentNewsCrawler.pl");	# torrent.newsbot.pl
	

	# newsarchive cachen
	# system("perl /root/zoozle/caching/NewsarchiveCachePage.pl");

#	&CACHE();
	
	# sitemaps fr heute erstellen
#	system("perl /root/zoozle/sitemaps/SitemapToday.pl");

	# sitemaps aus den suchkeywords der user bilden 
#	system("perl /root/zoozle/sitemaps/SitemapKeywordsToday.pl");
		
	system("date");	
	sleep 21600;

};


sub CACHE(){

	my $timeout	= "300";
	my $ua		= "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.6) Gecko/20040206 Firefox/1.0.1";

	my %CACHING = (
		"/server/wwwroot/index.html" => "http://www.zoozle.org/index.php",		# .net: "/srv/server/wwwroot/index.html" => "http://www.zoozle.net/index.php",
	#	"/server/en.wwwroot/index.html" => "http://en.zoozle.net/index.php",
	#	"/srv/server/de.wwwroot/index.html" => "http://de.zoozle.net/index.php",

	#	"/server/apache2/htdocs/de/index.html" => "http://85.214.90.176/de/index_de.php",
	#	"/server/apache2/htdocs/org/index.html" => "http://85.214.90.176/org/index_org.php",
	#	"/server/apache2/htdocs/devu/index.html" => "http://85.214.90.176/devu/index_devu.php"
	);

	while( my ( $path, $url ) = each(%CACHING) ){
		my $page = &http_get($url, $timeout, $ua, );
		print "Generate Cache for $url\n";
		
		if ( length($page) >= 1000 ) {

... [truncated, 15 more lines] ...

#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";
require "/root/zoozle/new-bots-X/functions.pl";

# renicing to 20 !
system("renice 20 $$");
system("clear");

# module laden
use strict;
no strict "refs";
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;


my $torrentflatfile		= "/root/zoozle/new-bots-X/torrent_news.txt";
unlink $torrentflatfile;

# filedonkey emule

my %LinksToScanDE = (		# 27 torrents
		"http://bitreactor.to/index"											=> '\/torrent\/',			# http://bitreactor.to/torrent/592734-filme-divx-xvid-otac-na-sluzbenom-putu-1985

... [truncated, 289 more lines] ...

#!/usr/bin/perl -I/root/zoozle/torrentbot/modules

my $VERION = "0.2";
require "/root/zoozle/new-bots-X/functions.pl";

# renicing to 20 !
system("renice 20 $$");
system("clear");

# module laden
use strict;
no strict "refs";
use IO::Handle;
use TorrentBot::HTTP::Get;
use TorrentBot::HTTP::Head;
use TorrentBot::LINK::ExtractorMem;
use Data::Dumper;

# objekte initialisieren
my $HEAD		= TorrentBot::HTTP::Head->new();
my $GET			= TorrentBot::HTTP::Get->new();
my $LINK		= TorrentBot::LINK::ExtractorMem->new();


# speicherschnittstellen initialisieren
my $AlreadSeenLinks;
my %AlreadSeenLinks		= ();
my @StatusCode			= ();
my @HOSTNAME			= ();
my @LinksToScan			= ();

my $StatusCode;
my $HOSTNAME;
my $TorrentDB;
my $TempStorePath;
my $BadUrls;
my $StatusFile;
my $pid;

my $StayOnSameHost		= 1;
my $AlreadyScannedLinks	= 0;
my $BadUrlCount			= 0;


my $torrentflatfile		= "/root/zoozle/new-bots-X/titanmule_test.txt";
unlink $torrentflatfile;

# filedonkey emule

my %LinksToScanDE = (		# 27 torrents

... [truncated, 183 more lines] ...