trendcrawler - Crawler - SOURCE CODE

http://search.cpan.org/~agent/WWW-Baidu-0.06/lib/WWW/Baidu.pm

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::Basic;
use Crawler::Config;

use strict;
use LWP::UserAgent;
use HTTP::Request::Common qw(GET);
use HTTP::Cookies;
use LWP::Simple;
use HTTP::Status ();
use File::Path qw(make_path);
use HTML::StripTags qw(strip_tags);
use HTML::Entities;
use URI;

my %AlreadyScanned = ();
my $AlreadyScanned;

my $D;
my %D = ();

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.2.0  - 2013-09-28
### Contact: sebastian.enger@gmail.com
### Aufgabe: Konstruktor
###########################
sub new {
	my ($class) = @_;
	
	my $config = Crawler::Config->new();
	my $self = {
	#	'kontostand' => $inKontostand,
		'config' => $config,
	};
	bless $self, $class;
  
  return $self;
}

... [truncated, 248 more lines] ...

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::Bing;
### http://search.cpan.org/~dhouston/Bing-Search/lib/Bing/Search.pm

use strict;
use Bing::Search;
use Bing::Search::Source::Web;

use Crawler::Config;
use Crawler::Basic;
	
# Konstruktor
sub new {
	
	my ($class) = @_;
	my $config 		= Crawler::Config->new();
	my $MemStore	= Crawler::MemStore->new();
	my $basic  		= Crawler::Basic->new();
	
	my $self = {
		'db_flat'	=> $MemStore,
		'config' => $config,
		'basic' => $basic,
	};
	bless $self, $class;
  return $self;
}

sub search(){
	
	my $self 	= shift;
	my $query 	= shift;
	
	my @return 	= ();
	my $basic 	= $self->{basic};
	
	my $search 	= Bing::Search->new(
      AppId => $self->{cfgValue},
      Query => $query,
    );
    

... [truncated, 24 more lines] ...

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::Config;
### http://search.cpan.org/~sherzodr/Config-Simple-4.59/Simple.pm

use strict;
use Config::Simple;
use File::Path;

my $ScannedFile = "/home/thecerial/trendcrawler/logfile/alreadyscanned.txt";
my $CookieFile = "/home/thecerial/trendcrawler/logfile/cookies.txt";

 
# Konstruktor
sub new {

	my ($class) = @_;
	
	my $cfgFile;
	if ($class !~ /\w{3,}/ig){
		$cfgFile = "/home/thecerial/trendcrawler/trendcrawler.cfg";
	}

	my $cfg = Config::Simple->new(filename=>$cfgFile);
	#%Config = $cfg->vars();
	
	my $self = {
		'config' => $cfg,
	#	'dispo' => $inDispo
	};
	bless $self, $class;
	make_path '/home/thecerial/trendcrawler/logfile/', {owner=>'thecerial', group=>'users'};

	return $self;
  
}

sub read() {
	
	my $self 	= shift;
	my $key		= shift;
	

... [truncated, 5 more lines] ...

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::DB;
# 
use strict;
use Crawler::Config;

use DBI;

# Konstruktor
sub new {
	
	my ($class) 	= @_;
	
	my $config 		= Crawler::Config->new();
	my $isRead 		= $config->read("Database");
	
	my $database 	= $isRead->{database};
	my $user 		= $isRead->{user};
	my $password 	= $isRead->{password};
	my $host 		= $isRead->{host};
	my $table 		= $isRead->{table};
	
	my $dbh = DBI->connect("dbi:mysql:dbname=$database",$user,$password, {
		AutoCommit => 1,#0=$dbh->commit(); 
		RaiseError => 1,
	  });
  
	my $self = {
		'dbh' => $dbh,
		'config' => $config,
		'db_table' => $table,
	};
	# connect to database on class initialization
	# Create table $today_trends (2013-09-27_trends) bei jedem start neu
	
	bless $self, $class;
  return $self;
}


sub save(){

... [truncated, 35 more lines] ...

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::GeneralCrawler;

use strict;
use HTML::LinkExtractor;
### 
use Config::Simple;
use Crawler::Basic;
use Crawler::DB;
 
 
# Konstruktor
sub new {
	my ($class) = @_;
	my $basic = Crawler::Basic->new();
	my $self = {
		'basic' => $basic,
	};
	bless $self, $class;
  return $self;
}

sub crawl(){

	my $self 				= shift;
	my $starturl 			= shift;
	
	my $content 			= $self->{basic}->fetch($starturl);
	my $arref_links 		= $self->{basic}->links($content,$starturl);
	
	my $text_content 		= $self->{basic}->stripHtml($content);
	my $arref_text_cloud 	= $self->{basic}->makeCloud($text_content,$starturl);
	
	# Google, Bing, Baidu, Yandex Web Search > get first 5 webpages and crawl them
	# Google News Search - Deutschland
	# Youtube Search
	# Patent DB Search -> grap information
	# Bachelor und Masterarbeiten Search
	# Twitter Search for keywords
	# Collect Data and save to database: -> $links_arref ins sql->links speichern
	

... [truncated, 4 more lines] ...

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::Google;
# http://search.cpan.org/~rokr/Google-Search-0.028/lib/Google/Search.pm

use strict;
use Google::Search;

use Crawler::Config;
use Crawler::MemStore;
use Crawler::Basic;

# Konstruktor
sub new {
	
	my ($class) 	= @_;
	
	my $config 		= Crawler::Config->new();
	my $MemStore	= Crawler::MemStore->new();
	my $basic  		= Crawler::Basic->new();
	
	my $self = {
		'db_flat'	=> $MemStore,
		'config' => $config,
		'basic' => $basic,
	};
	bless $self, $class;
  return $self;
}

sub news(){

}

sub patent(){

}

sub search(){
	
	my $self = shift;
	my $query = shift;

... [truncated, 25 more lines] ...

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::MemStore;
### http://perldoc.perl.org/Storable.html

use strict;
use Storable;
use File::Path;
use DB_File;
use Fcntl;

# Konstruktor
sub new {

	my ($cfgFile) = @_;
	if ($cfgFile !~ /\w{3,}/ig){
		$cfgFile = "/home/thecerial/trendcrawler/trendcrawler.db";
	}

	my ($DB,%DB);
	tie (%DB, DB_File, $cfgFile, O_RDWR|O_CREAT, 0644, $DB_HASH);
	
#	my %Store = ();
#	store \%Store, $cfgFile;
#	my $hashref = retrieve($cfgFile);
	
	my $self = {
	#	'store' => $hashref,
		'db_flat' => $DB,
	};
	bless $self, $class;
	make_path '/home/thecerial/trendcrawler/', {owner=>'thecerial', group=>'users'};

	return $self;
}

sub empty(){

	my $self = shift;
	
	delete $self->{db_flat}{googlelinks};
	delete $self->{db_flat}{binglinks};

... [truncated, 16 more lines] ...

#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.2  - 2013-09-28
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::Twitter;

use strict;
use Net::Twitter;
use Net::Twitter::Search;
use Scalar::Util 'blessed';
 
 
# Konstruktor
sub new {
	my ($class) = @_;
	my $basic = Crawler::Basic->new();
	my $self = {
		'basic' => $basic,
	};
	bless $self, $class;
  return $self;
}


http://search.cpan.org/~mmims/Net-Twitter-4.00007/lib/Net/Twitter.pod






# https://twitter.com/search?q=perl%20programming%20language%20lang%3Ade&src=typd&f=realtime

# Just the Search API; exceptions thrown on error
$nt = Net::Twitter->new(traits => [qw/API::Search/]);
eval {
   # Parameters: q, callback, lang, rpp, page, since_id, geocode, show_user
   my $r = $nt->search({
      q=>"\"easter service\" OR \"sunrise service\"",
      lang=>"en",
      geocode=>"",
      rpp=>50,
      since=>"2013-01-01"
   });

... [truncated, 16 more lines] ...

http://search.cpan.org/~arto/WWW-Search-Yandex/lib/WWW/Search/Yandex.pm