#!/usr/bin/perl

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: Commercial
### Version: 1.1.1  - 2013-09-18
### Contact: sebastian.enger@gmail.com
###########################

package Crawler::GeneralCrawler;

use strict;
use HTML::LinkExtractor;
### 
use Config::Simple;
use Crawler::Basic;
use Crawler::DB;
 
 
# Konstruktor
sub new {
	my ($class) = @_;
	my $basic = Crawler::Basic->new();
	my $self = {
		'basic' => $basic,
	};
	bless $self, $class;
  return $self;
}

sub crawl(){

	my $self 				= shift;
	my $starturl 			= shift;
	
	my $content 			= $self->{basic}->fetch($starturl);
	my $arref_links 		= $self->{basic}->links($content,$starturl);
	
	my $text_content 		= $self->{basic}->stripHtml($content);
	my $arref_text_cloud 	= $self->{basic}->makeCloud($text_content,$starturl);
	
	# Google, Bing, Baidu, Yandex Web Search > get first 5 webpages and crawl them
	# Google News Search - Deutschland
	# Youtube Search
	# Patent DB Search -> grap information
	# Bachelor und Masterarbeiten Search
	# Twitter Search for keywords
	# Collect Data and save to database: -> $links_arref ins sql->links speichern
	
}


