#!/usr/local/bin/perl -w
#
# validate with http://www.searchengineworld.com/cgi-bin/robotcheck.cgi
#

my $ua_db = '/usr/domains/d12world.com/html/robots.ua.db';
my $cl_db = '/usr/domains/d12world.com/html/robots.cl.db';

print "Content-type: text/plain\n\n";
my $ua = ($ENV{'HTTP_USER_AGENT'}) ? $ENV{'HTTP_USER_AGENT'} : "NO-USER-AGENT";
print "# Hello $ua\n";
print "User-agent: *\n";

open(UA,"<$ua_db")
	|| die "Cannot open $ua_db: $!";

my ($rx,$cl,$cdu,$ot);

while (<UA>) {
	# useragent:crawldelay:class:other[,other[,other...]]
	($rx,$cdu,$cl,$ot) = split(/:/,$_);
	next if ($rx =~ /^#/);
	last if ($ua =~ m/$rx/);
}

close(UA);

open(CL,"<$cl_db")
	|| die "Cannot open $cl_db: $!";

my ($cx,$cdc,$da);

while (<CL>) {
	# class:crawldelay:disallow[,disallow[,disallow...]]
	($cx,$cdc,$da) = split(/:/,$_);
	last if ($cl eq $cx);
}

close(CL);

my $cd = ($cdc < $cdu) ? $cdu : $cdc;

print "Crawl-delay: $cd\n";

foreach (split(/,/,$da)) {
	chomp;
	next unless($_);
	$_ = s/\/board\//\//;
	print "Disallow: $_\n";
}
print "Disallow: /private/\n";

foreach (split/,/,$ot) {
	chomp;
	next unless($_);
	print "$_\n";
}

print "# Goodbye\n";