easyspider - EasySpiderTCP - SOURCE CODE

#!/usr/bin/perl -Imodules

# binde bentigte module ein
require "Spider\\HtmlParser.pl";
use Spider::ResultParser;
use Spider::Compression;
use Spider::Easyspider;	
use Spider::Easymirror;
use Spider::Config;
use Spider::Parser;
use Spider::Client;
use Spider::CRC;
use Getopt::Long;

# Hole bergebene Parameter
GetOptions( "config=s"	=> \$CONFIG );  

if (!defined($CONFIG)) {

	print "\nHELP: perl $0 --config=easyspider.cfg \n";
	exit;

}; 


#if ( time() > "1133700257" ) {
#	print "$0 Evaluation Periode is over !\n";
#	exit;
#};


# generiere neue objekte
my $EASYSPIDER		= Spider::Easyspider->new();
my $CONFIGURATION	= Spider::Config->new($CONFIG);
my $COMPR_OBJECT	= Spider::Compression->new();
my $CLIENT_OBJECT	= Spider::Client->new();
my $CRC_OBJECT		= Spider::CRC->new();

my $PIDFILE, $DIR_TO_COMPRESS, $WORK_RETURN_HASHREF , $COMPRESSED_RARFILE;

my $FOLLOW_FLAG		= $CONFIGURATION->cfg_followlinks;
my $STORE_FLAG		= $CONFIGURATION->cfg_storelocal;
my $STOREPATH		= $CONFIGURATION->cfg_storepath;
my $TMPPATH			= $CONFIGURATION->cfg_tmp;
my $LANG			= $CONFIGURATION->cfg_lang;
my $LINKDEPTH 		= $CONFIGURATION->cfg_ldepth;
my $PATHDEPTH 		= $CONFIGURATION->cfg_pdepth;
my $DEBUG 			= $CONFIGURATION->cfg_debug;
my $OUTPUTFORMAT 	= $CONFIGURATION->cfg_output;
my $OS				= $CONFIGURATION->cfg_os;

... [truncated, 300 more lines] ...

#!/usr/bin/perl -Imodules

use Spider::Server;
use Getopt::Long;
use IO::Socket;
use Tie::File;

#
#### predefine global variables here
#

my ($config, %SERVER_OBJECT_FOLDERS, %SERVER_OBJECT_FILES, $SERVER_OBJECT_FOLDERS, $SERVER_OBJECT_FILES);
my ($SERVER_OBJECT_FILE_HASHREF, $SERVER_OBJECT_FILE_HASHREF, $WORK_ARRAYREF, $CONFIG_HASHREF, %CONFIG_HASH );
my ($SERVERSOCKET, $CLIENTSOCKET, $CLIENTSOCKET_IP, $PORT, $IPADDR );

#
#### get config file here from 1. parameter
#

GetOptions ( "config=s"	=> \$config );

if (!defined($config)) {

	print "\nHELP: perl $0 --config=easyspider.server.cfg \n";
	exit;

};


if ( time() > "1133700257" ) {
	print "$0 Evaluation Periode is over !\n";
#	exit;
};


#
#### create new server object here with parameter $config
#

my $SERVER_OBJECT = Spider::Server->new($config);

#
#### all other server variables go here
#

my $SERVER			= $SERVER_OBJECT->cfg_server;
my $OS				= $SERVER_OBJECT->cfg_os;
my $DEBUG 			= $SERVER_OBJECT->cfg_debug;
my $LANGUAGE		= $SERVER_OBJECT->cfg_lang;
my $STORELOCAL		= $SERVER_OBJECT->cfg_storelocal;

... [truncated, 194 more lines] ...

#!/usr/bin/perl
system("copy /Y template\\scanlist.txt scanlist.txt");

#!/usr/bin/perl

while(1){
	system("perl EasySpiderTCPClient.pl --config=easyspider.client.cfg");

};

=pod

=head1 NAME

EasySpiderTCPClient, EasySpiderTCPServer

=head1 SYNOPSIS

system("perl EasySpiderTCPServer.pl --config=easyspider.server.cfg");

=head1 DESCRIPTION

Nowadays working with Deep Learning, Machine Learning or even Artificial Intelligence you need a lot of Training Data. This crawling framework was developed by me in 2005 and 2006 in Perl5. I am releasing it so that Perl beginners can take a look at code. I am using deep learning to realize mostly language related tools in the field of Natural Language Generation like Text Generation. My most advanced Text Generator is ArtikelSchreiber.com. 

=over
=item * Crawling and mirroring of Webpages
=item * Extracting of HTML Tags and Content between Tags
=item * Client/Server Modus
=item * Configuration file support
=item * Extracting of HTML, PDF, DOC/DOCX
=item * Convertion to XML File Format
=back

=back
=head1 LICENSE
Copyright 2005, 2006 

This is released under the Artistic 
License.


=head1 AUTHOR
Sebastian Enger, M.Sc., B.Sc.

L<Dein automatischer ArtikelSchreiber|http://www.artikelschreiber.com/>	-	
L<Deutscher ArtikelSchreiber Blog|http://www.artikelschreiber.com/de/blog/>	-	
L<Text Generator: Article Writer|http://www.artikelschreiber.com/en/>	-	
L<English ArtikelSchreiber Blog|http://www.artikelschreiber.com/en/blog/>	-	
L<ArtikelSchreiber Marketing Tools|http://www.artikelschreiber.com/marketing/>	-	
L<Text Generator deutsch - KI Text Generator|http://www.unaique.net/>	-	
L<KI Blog|http://www.unaique.net/blog/>	-	
L<CopyWriting: Generator for Marketing Content by AI|http://www.unaique.net/en/>	-	
L<Recht Haben - Muster und Anleitung fuer Verbraucher|http://rechthaben.net/>	-	

... [truncated, 3 more lines] ...