byterazor/qpsmtpd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perltidy -b watch summarize show_message log2sql

Browse Source
This commit is contained in:
Matt Simerson 2013-04-21 00:54:43 -04:00
parent 9c96ae78e4
commit 610c39dc74
4 changed files with 540 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

530
log/log2sql
View File

@ -22,11 +22,11 @@ my (%plugins, %os, %message_ids);
my $has_cleanup; my $has_cleanup;
my $db = get_db(); my $db = get_db();
foreach my $file ( @logfiles ) { foreach my $file (@logfiles) {
my ($fid, $offset) = check_logfile($file); my ($fid, $offset) = check_logfile($file);
$fid or next; $fid or next;
parse_logfile( $file, $fid, $offset ); parse_logfile($file, $fid, $offset);
}; }
exit; exit;
@ -47,14 +47,14 @@ sub trim_message {
return '' if $mess eq 'TLS setup returning'; return '' if $mess eq 'TLS setup returning';
return $mess; return $mess;
}; }
sub get_os_id { sub get_os_id {
my $p0f_string = shift or return; my $p0f_string = shift or return;
$p0f_string =~ s/\s+$//; $p0f_string =~ s/\s+$//;
$p0f_string =~ s/^\s+//; $p0f_string =~ s/^\s+//;
return if ! $p0f_string; return if !$p0f_string;
return if $p0f_string =~ /no match/; return if $p0f_string =~ /no match/;
return if $p0f_string =~ /^skip/; return if $p0f_string =~ /^skip/;
return if $p0f_string =~ /^\d/; return if $p0f_string =~ /^\d/;
@ -62,266 +62,267 @@ sub get_os_id {
return if $p0f_string !~ /\w/; return if $p0f_string !~ /\w/;
return if $p0f_string =~ /no longer in the cache/; return if $p0f_string =~ /no longer in the cache/;
if ( ! scalar keys %os ) { if (!scalar keys %os) {
my $ref = exec_query( 'SELECT * FROM os' ); my $ref = exec_query('SELECT * FROM os');
foreach my $o ( @$ref ) { foreach my $o (@$ref) {
$os{ $o->{name} } = $o->{id}; $os{$o->{name}} = $o->{id};
}; }
}; }
if ( ! defined $os{$p0f_string} ) { if (!defined $os{$p0f_string}) {
warn "missing OS for $p0f_string\n"; warn "missing OS for $p0f_string\n";
}; }
return $os{$p0f_string}; return $os{$p0f_string};
}; }
sub get_plugin_id { sub get_plugin_id {
my $plugin = shift; my $plugin = shift;
if ( ! scalar keys %plugins ) { if (!scalar keys %plugins) {
my $ref = exec_query( 'SELECT * FROM plugin' ); my $ref = exec_query('SELECT * FROM plugin');
foreach my $p ( @$ref ) { foreach my $p (@$ref) {
$plugins{ $p->{name} } = $p->{id}; $plugins{$p->{name}} = $p->{id};
$plugins{ $p->{id} } = $p->{name}; $plugins{$p->{id}} = $p->{name};
}; }
$ref = exec_query( 'SELECT * FROM plugin_aliases' ); $ref = exec_query('SELECT * FROM plugin_aliases');
foreach my $pa ( @$ref ) { foreach my $pa (@$ref) {
$plugins{ $pa->{name} } = $pa->{plugin_id}; $plugins{$pa->{name}} = $pa->{plugin_id};
}; }
}; }
if (!defined $plugins{$plugin}) {
if ( ! defined $plugins{$plugin} ) {
#warn Dumper(\%plugins); #warn Dumper(\%plugins);
die "missing DB plugin $plugin\n"; die "missing DB plugin $plugin\n";
}; }
return $plugins{$plugin}; return $plugins{$plugin};
}; }
sub get_msg_id { sub get_msg_id {
my ( $fid, $pid ) = @_; my ($fid, $pid) = @_;
return $message_ids{ "$fid-$pid" } if $message_ids{ "$fid-$pid" }; return $message_ids{"$fid-$pid"} if $message_ids{"$fid-$pid"};
#print "searching for message $pid..."; #print "searching for message $pid...";
my $msgs = exec_query( my $msgs = exec_query('SELECT * FROM message WHERE file_id=? AND qp_pid=?',
'SELECT * FROM message WHERE file_id=? AND qp_pid=?', [$fid, $pid]);
[ $fid, $pid ]
);
#print scalar @$msgs ? "y\n" : "n\n"; #print scalar @$msgs ? "y\n" : "n\n";
if ( $msgs->[0]{id} ) { if ($msgs->[0]{id}) {
$message_ids{ "$fid-$pid" } = $msgs->[0]{id}; $message_ids{"$fid-$pid"} = $msgs->[0]{id};
}; }
return $msgs->[0]{id}; return $msgs->[0]{id};
}; }
sub create_message { sub create_message {
my ( $fid, $ts, $pid, $message ) = @_; my ($fid, $ts, $pid, $message) = @_;
my ($host, $ip) = split /\s/, $message; my ($host, $ip) = split /\s/, $message;
$ip = substr $ip, 1, -1; # remove brackets $ip = substr $ip, 1, -1; # remove brackets
my $id = exec_query( my $id = exec_query(
"INSERT INTO message SET file_id=?, connect_start=FROM_UNIXTIME(?), qp_pid=?, ip=INET_ATON(?)", "INSERT INTO message SET file_id=?, connect_start=FROM_UNIXTIME(?), qp_pid=?, ip=INET_ATON(?)",
[ $fid, $ts, $pid, $ip ] [$fid, $ts, $pid, $ip]
); );
if ( $host && $host ne 'Unknown' ) { if ($host && $host ne 'Unknown') {
exec_query( "UPDATE message SET hostname=? WHERE id=?", [ $host, $id ] ); exec_query("UPDATE message SET hostname=? WHERE id=?", [$host, $id]);
}; }
#warn "host updated: $host\n"; #warn "host updated: $host\n";
}; }
sub insert_plugin { sub insert_plugin {
my ( $msg_id, $plugin, $message ) = @_; my ($msg_id, $plugin, $message) = @_;
my $plugin_id = get_plugin_id( $plugin ); my $plugin_id = get_plugin_id($plugin);
if ( $plugin eq 'ident::geoip' ) { if ($plugin eq 'ident::geoip') {
my ($gip, $distance) = $message =~ /(.*?),\s+([\d]+)\skm/; my ($gip, $distance) = $message =~ /(.*?),\s+([\d]+)\skm/;
if ( $distance ) { if ($distance) {
exec_query( 'UPDATE message SET distance=? WHERE id=?', [ $distance, $msg_id ] ); exec_query('UPDATE message SET distance=? WHERE id=?',
[$distance, $msg_id]);
$message = $gip; $message = $gip;
} }
} }
elsif ( $plugin =~ /^ident::p0f/ ) { elsif ($plugin =~ /^ident::p0f/) {
my $os_id = get_os_id( $message ); my $os_id = get_os_id($message);
if ( $os_id ) { if ($os_id) {
exec_query( 'UPDATE message SET os_id=? WHERE id=?', [ $os_id, $msg_id ] ); exec_query('UPDATE message SET os_id=? WHERE id=?',
[$os_id, $msg_id]);
$message = 'pass'; $message = 'pass';
} }
} }
elsif ( $plugin eq 'connection_time' ) { elsif ($plugin eq 'connection_time') {
my ($seconds) = $message =~ /\s*([\d\.]+)\s/; my ($seconds) = $message =~ /\s*([\d\.]+)\s/;
if ( $seconds ) { if ($seconds) {
exec_query( 'UPDATE message SET time=? WHERE id=?', [ $seconds, $msg_id ] ); exec_query('UPDATE message SET time=? WHERE id=?',
[$seconds, $msg_id]);
$message = 'pass'; $message = 'pass';
} }
} }
my $result = get_score( $message ); my $result = get_score($message);
if ( $result ) { if ($result) {
$message = trim_message($message); $message = trim_message($message);
}; }
exec_query( 'INSERT INTO message_plugin SET msg_id=?, plugin_id=?, result=?, string=?', exec_query(
[ $msg_id, $plugin_id, $result, $message ] 'INSERT INTO message_plugin SET msg_id=?, plugin_id=?, result=?, string=?',
[$msg_id, $plugin_id, $result, $message]
); );
}; }
sub parse_logfile { sub parse_logfile {
my $file = shift; my $file = shift;
my $fid = shift; my $fid = shift;
my $offset = shift || 0; my $offset = shift || 0;
my $path = "$logdir/$file"; my $path = "$logdir/$file";
print "parsing file $file (id: $fid) from offset $offset\n"; print "parsing file $file (id: $fid) from offset $offset\n";
open my $F, '<', $path or die "could not open $path: $!"; open my $F, '<', $path or die "could not open $path: $!";
seek( $F, $offset, 0 ) if $offset; seek($F, $offset, 0) if $offset;
while ( defined (my $line = <$F> ) ) { while (defined(my $line = <$F>)) {
chomp $line; chomp $line;
next if ! $line; next if !$line;
my ( $type, $pid, $hook, $plugin, $message ) = parse_line( $line ); my ($type, $pid, $hook, $plugin, $message) = parse_line($line);
next if ! $type; next if !$type;
next if $type eq 'info'; next if $type eq 'info';
next if $type eq 'unknown'; next if $type eq 'unknown';
next if $type eq 'response'; next if $type eq 'response';
next if $type eq 'init'; # doesn't occur in all deployment models next if $type eq 'init'; # doesn't occur in all deployment models
next if $type eq 'cleanup'; next if $type eq 'cleanup';
next if $type eq 'error'; next if $type eq 'error';
my $ts = tai2unix( (split /\s/, $line)[0] ); # print "ts: $ts\n"; my $ts = tai2unix((split /\s/, $line)[0]); # print "ts: $ts\n";
my $msg_id = get_msg_id( $fid, $pid ) or do { my $msg_id = get_msg_id($fid, $pid) or do {
create_message( $fid, $ts, $pid, $message ) if $type eq 'connect'; create_message($fid, $ts, $pid, $message) if $type eq 'connect';
next; next;
}; };
#warn "type: $type\n"; #warn "type: $type\n";
if ( $type eq 'plugin' ) { if ($type eq 'plugin') {
next if $plugin eq 'naughty'; # housekeeping only next if $plugin eq 'naughty'; # housekeeping only
insert_plugin( $msg_id, $plugin, $message ); insert_plugin($msg_id, $plugin, $message);
} }
elsif ( $type eq 'queue' ) { elsif ($type eq 'queue') {
exec_query('UPDATE message SET result=? WHERE id=?', [ 3, $msg_id ] ); exec_query('UPDATE message SET result=? WHERE id=?', [3, $msg_id]);
} }
elsif ( $type eq 'reject' ) { elsif ($type eq 'reject') {
exec_query('UPDATE message SET result=? WHERE id=?', [ -3, $msg_id ] ); exec_query('UPDATE message SET result=? WHERE id=?', [-3, $msg_id]);
} }
elsif ( $type eq 'close' ) { elsif ($type eq 'close') {
if ( $message eq 'Connection Timed Out' ) { if ($message eq 'Connection Timed Out') {
exec_query('UPDATE message SET result=? WHERE id=?', [ -1, $msg_id ] ); exec_query('UPDATE message SET result=? WHERE id=?',
}; [-1, $msg_id]);
}
elsif ( $type eq 'connect' ) { }
elsif ( $type eq 'dispatch' ) {
if ( substr($message, 0, 21) eq 'dispatching MAIL FROM' ) {
my ($from) = $message =~ /<(.*?)>/;
exec_query('UPDATE message SET mail_from=? WHERE id=?', [ $from, $msg_id ] );
} }
elsif ( substr($message, 0, 19) eq 'dispatching RCPT TO' ) { }
my ($to) = $message =~ /<(.*?)>/; elsif ($type eq 'connect') { }
exec_query('UPDATE message SET rcpt_to=? WHERE id=? AND rcpt_to IS NULL', [ $to, $msg_id ] ); elsif ($type eq 'dispatch') {
if (substr($message, 0, 21) eq 'dispatching MAIL FROM') {
my ($from) = $message =~ /<(.*?)>/;
exec_query('UPDATE message SET mail_from=? WHERE id=?',
[$from, $msg_id]);
} }
elsif ( $message =~ m/dispatching (EHLO|HELO) (.*)/ ) { elsif (substr($message, 0, 19) eq 'dispatching RCPT TO') {
exec_query('UPDATE message SET helo=? WHERE id=?', [ $2, $msg_id ] ); my ($to) = $message =~ /<(.*?)>/;
exec_query(
'UPDATE message SET rcpt_to=? WHERE id=? AND rcpt_to IS NULL',
[$to, $msg_id]
);
} }
elsif ( $message eq 'dispatching DATA' ) { } elsif ($message =~ m/dispatching (EHLO|HELO) (.*)/) {
elsif ( $message eq 'dispatching QUIT' ) { } exec_query('UPDATE message SET helo=? WHERE id=?',
elsif ( $message eq 'dispatching STARTTLS' ) { } [$2, $msg_id]);
elsif ( $message eq 'dispatching RSET' ) { } }
elsif ($message eq 'dispatching DATA') { }
elsif ($message eq 'dispatching QUIT') { }
elsif ($message eq 'dispatching STARTTLS') { }
elsif ($message eq 'dispatching RSET') { }
else { else {
# anything here is likely an unrecognized command # anything here is likely an unrecognized command
#print "$message\n"; #print "$message\n";
}; }
} }
else { else {
print "$type $pid $hook $plugin $message\n"; print "$type $pid $hook $plugin $message\n";
}; }
}; }
close $F; close $F;
}; }
sub check_logfile { sub check_logfile {
my $file = shift; my $file = shift;
my $path = "$logdir/$file"; my $path = "$logdir/$file";
die "missing file $logdir/$file" if ! -f "$logdir/$file"; die "missing file $logdir/$file" if !-f "$logdir/$file";
my $inode = stat($path)->ino or die "unable to get inode for $path\n"; my $inode = stat($path)->ino or die "unable to get inode for $path\n";
my $size = stat($path)->size or die "unable to get size for $path\n"; my $size = stat($path)->size or die "unable to get size for $path\n";
my $exists; my $exists;
#warn "check if file $file is in the DB as 'current'\n"; #warn "check if file $file is in the DB as 'current'\n";
if ( $file =~ /^\@/ ) { if ($file =~ /^\@/) {
$exists = exec_query( $exists = exec_query('SELECT * FROM log WHERE inode=? AND name=?',
'SELECT * FROM log WHERE inode=? AND name=?', [$inode, 'current']);
[ $inode, 'current' ] if (@$exists) {
);
if ( @$exists ) {
print "Updating current -> $file\n"; print "Updating current -> $file\n";
exec_query( exec_query('UPDATE log SET name=? WHERE inode=? AND name=?',
'UPDATE log SET name=? WHERE inode=? AND name=?', [$file, $inode, 'current']);
[ $file, $inode, 'current' ] return ($exists->[0]{id}, $exists->[0]{size}); # continue parsing
); }
return ( $exists->[0]{id}, $exists->[0]{size} ); # continue parsing }
};
};
if ( $file eq 'current' ) { if ($file eq 'current') {
$exists = exec_query( $exists = exec_query('SELECT * FROM log WHERE inode=? AND name=?',
'SELECT * FROM log WHERE inode=? AND name=?', [$inode, $file]);
[ $inode, $file ] if (@$exists) {
); exec_query('UPDATE log SET size=? WHERE inode=? AND name=?',
if ( @$exists ) { [$size, $inode, 'current']);
exec_query( return ($exists->[0]{id}, $exists->[0]{size}); # continue parsing
'UPDATE log SET size=? WHERE inode=? AND name=?', }
[ $size, $inode, 'current' ] }
);
return ( $exists->[0]{id}, $exists->[0]{size} ); # continue parsing
};
};
$exists = exec_query( $exists =
'SELECT * FROM log WHERE name=? AND size=?', exec_query('SELECT * FROM log WHERE name=? AND size=?', [$file, $size]);
[ $file, $size ]
);
return if @$exists; # log file hasn't changed, ignore it return if @$exists; # log file hasn't changed, ignore it
#print Dumper($exists); #print Dumper($exists);
# file is a new one we haven't seen, add to DB and parse # file is a new one we haven't seen, add to DB and parse
my $id = exec_query( my $id = exec_query(
'INSERT INTO log SET inode=?, size=?, name=?, created=FROM_UNIXTIME(?)', 'INSERT INTO log SET inode=?, size=?, name=?, created=FROM_UNIXTIME(?)',
[ $inode, $size, $file, stat($path)->ctime ] [$inode, $size, $file, stat($path)->ctime]
); );
print "new file id: $id\n"; print "new file id: $id\n";
return ( $id ); return ($id);
}; }
sub get_log_dir { sub get_log_dir {
if ( -d "log/main" ) { if (-d "log/main") {
my $wd = Cwd::cwd(); my $wd = Cwd::cwd();
return "$wd/log/main"; return "$wd/log/main";
}; }
foreach my $user ( qw/ qpsmtpd smtpd / ) { foreach my $user (qw/ qpsmtpd smtpd /) {
my ($homedir) = (getpwnam( $user ))[7] or next; my ($homedir) = (getpwnam($user))[7] or next;
if ( -d "$homedir/log" ) { if (-d "$homedir/log") {
return "$homedir/log/main"; return "$homedir/log/main";
}; }
if ( -d "$homedir/smtpd/log" ) { if (-d "$homedir/smtpd/log") {
return "$homedir/smtpd/log/main"; return "$homedir/smtpd/log/main";
}; }
}; }
}; }
sub get_logfiles { sub get_logfiles {
my $dir = shift; my $dir = shift;
@ -329,134 +330,159 @@ sub get_logfiles {
opendir my $D, $dir or die "unable to open log dir $dir\n"; opendir my $D, $dir or die "unable to open log dir $dir\n";
my @files; my @files;
while ( defined( my $f = readdir($D) ) ) { while (defined(my $f = readdir($D))) {
next if ! -f "$dir/$f"; # ignore anything that's not a file next if !-f "$dir/$f"; # ignore anything that's not a file
if ( $f =~ /^\@.*s$/ ) { if ($f =~ /^\@.*s$/) {
push @files, $f; push @files, $f;
}; }
} }
push @files, "current"; # always have this one last push @files, "current"; # always have this one last
closedir $D; closedir $D;
return @files; return @files;
}; }
sub parse_line { sub parse_line {
my $line = shift; my $line = shift;
my ($tai, $pid, $message) = split /\s+/, $line, 3; my ($tai, $pid, $message) = split /\s+/, $line, 3;
return if ! $message; # garbage in the log file return if !$message; # garbage in the log file
# lines seen many times per connection # lines seen many times per connection
return parse_line_plugin( $line ) if substr($message, 0, 1) eq '('; return parse_line_plugin($line) if substr($message, 0, 1) eq '(';
return ( 'dispatch', $pid, undef, undef, $message ) if substr($message, 0, 12) eq 'dispatching '; return ('dispatch', $pid, undef, undef, $message)
return ( 'queue', $pid, undef, undef, $message ) if substr($message, 0, 11) eq '250 Queued!'; if substr($message, 0, 12) eq 'dispatching ';
return ( 'response', $pid, undef, undef, $message ) if $message =~ /^[2|3]\d\d/; return ('queue', $pid, undef, undef, $message)
if substr($message, 0, 11) eq '250 Queued!';
return ('response', $pid, undef, undef, $message)
if $message =~ /^[2|3]\d\d/;
# lines seen about once per connection # lines seen about once per connection
return ( 'init', $pid, undef, undef, $message ) if substr($message, 0, 19) eq 'Accepted connection'; return ('init', $pid, undef, undef, $message)
return ( 'connect', $pid, undef, undef, substr( $message, 16) ) if substr($message, 0, 15) eq 'Connection from'; if substr($message, 0, 19) eq 'Accepted connection';
return ( 'connect', $pid, undef, undef, substr( $message, 16) ) if substr($message, 0, 8) eq 'connect '; return ('connect', $pid, undef, undef, substr($message, 16))
return ( 'close', $pid, undef, undef, $message ) if substr($message, 0, 6) eq 'close '; if substr($message, 0, 15) eq 'Connection from';
return ( 'close', $pid, undef, undef, $message ) if $message eq 'Connection Timed Out'; return ('connect', $pid, undef, undef, substr($message, 16))
return ( 'close', $pid, undef, undef, $message ) if substr($message, 0, 20) eq 'click, disconnecting'; if substr($message, 0, 8) eq 'connect ';
return parse_line_cleanup( $line ) if substr($message, 0, 11) eq 'cleaning up'; return ('close', $pid, undef, undef, $message)
if substr($message, 0, 6) eq 'close ';
return ('close', $pid, undef, undef, $message)
if $message eq 'Connection Timed Out';
return ('close', $pid, undef, undef, $message)
if substr($message, 0, 20) eq 'click, disconnecting';
return parse_line_cleanup($line)
if substr($message, 0, 11) eq 'cleaning up';
# lines seen less than once per connection # lines seen less than once per connection
return ( 'info', $pid, undef, undef, $message ) if $message eq 'spooling message to disk'; return ('info', $pid, undef, undef, $message)
return ( 'reject', $pid, undef, undef, $message ) if $message =~ /^[4|5]\d\d/; if $message eq 'spooling message to disk';
return ( 'reject', $pid, undef, undef, $message ) if substr($message, 0, 14) eq 'deny mail from'; return ('reject', $pid, undef, undef, $message) if $message =~ /^[4|5]\d\d/;
return ( 'reject', $pid, undef, undef, $message ) if substr($message, 0, 18) eq 'denysoft mail from'; return ('reject', $pid, undef, undef, $message)
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 15) eq 'Lost connection'; if substr($message, 0, 14) eq 'deny mail from';
return ( 'info', $pid, undef, undef, $message ) if $message eq 'auth success cleared naughty'; return ('reject', $pid, undef, undef, $message)
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 15) eq 'Running as user'; if substr($message, 0, 18) eq 'denysoft mail from';
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 16) eq 'Loaded Qpsmtpd::'; return ('info', $pid, undef, undef, $message)
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 24) eq 'Permissions on spool_dir'; if substr($message, 0, 15) eq 'Lost connection';
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 13) eq 'Listening on '; return ('info', $pid, undef, undef, $message)
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 18) eq 'size_threshold set'; if $message eq 'auth success cleared naughty';
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 12) eq 'tls: ciphers'; return ('info', $pid, undef, undef, $message)
return ( 'error', $pid, undef, undef, $message ) if substr($message, 0, 22) eq 'of uninitialized value'; if substr($message, 0, 15) eq 'Running as user';
return ( 'error', $pid, undef, undef, $message ) if substr($message, 0, 8) eq 'symbol "'; return ('info', $pid, undef, undef, $message)
return ( 'error', $pid, undef, undef, $message ) if substr($message, 0, 9) eq 'error at '; if substr($message, 0, 16) eq 'Loaded Qpsmtpd::';
return ( 'error', $pid, undef, undef, $message ) if substr($message, 0, 15) eq 'Could not print'; return ('info', $pid, undef, undef, $message)
if substr($message, 0, 24) eq 'Permissions on spool_dir';
return ('info', $pid, undef, undef, $message)
if substr($message, 0, 13) eq 'Listening on ';
return ('info', $pid, undef, undef, $message)
if substr($message, 0, 18) eq 'size_threshold set';
return ('info', $pid, undef, undef, $message)
if substr($message, 0, 12) eq 'tls: ciphers';
return ('error', $pid, undef, undef, $message)
if substr($message, 0, 22) eq 'of uninitialized value';
return ('error', $pid, undef, undef, $message)
if substr($message, 0, 8) eq 'symbol "';
return ('error', $pid, undef, undef, $message)
if substr($message, 0, 9) eq 'error at ';
return ('error', $pid, undef, undef, $message)
if substr($message, 0, 15) eq 'Could not print';
print "UNKNOWN LINE: $line\n"; print "UNKNOWN LINE: $line\n";
return ( 'unknown', $pid, undef, undef, $message ); return ('unknown', $pid, undef, undef, $message);
}; }
sub parse_line_plugin { sub parse_line_plugin {
my ($line) = @_; my ($line) = @_;
# @tai 13486 (connect) ident::p0f: Windows (XP/2000 (RFC1323+, w, tstamp-)) # @tai 13486 (connect) ident::p0f: Windows (XP/2000 (RFC1323+, w, tstamp-))
# @tai 13681 (connect) dnsbl: fail, NAUGHTY # @tai 13681 (connect) dnsbl: fail, NAUGHTY
# @tai 15787 (connect) karma: pass, no penalty (0 naughty, 3 nice, 3 connects) # @tai 15787 (connect) karma: pass, no penalty (0 naughty, 3 nice, 3 connects)
# @tai 27500 (queue) queue::qmail_2dqueue: (for 27481) Queuing to /var/qmail/bin/qmail-queue # @tai 27500 (queue) queue::qmail_2dqueue: (for 27481) Queuing to /var/qmail/bin/qmail-queue
my ($tai, $pid, $hook, $plugin, $message ) = split /\s/, $line, 5; my ($tai, $pid, $hook, $plugin, $message) = split /\s/, $line, 5;
$plugin =~ s/:$//; $plugin =~ s/:$//;
return parse_line_plugin_p0f( $line ) if $plugin =~ /^ident::p0f/; return parse_line_plugin_p0f($line) if $plugin =~ /^ident::p0f/;
return parse_line_plugin_dspam( $line ) if $plugin =~ /^dspam/; return parse_line_plugin_dspam($line) if $plugin =~ /^dspam/;
return parse_line_plugin_spamassassin( $line ) if $plugin =~ /^spamassassin/; return parse_line_plugin_spamassassin($line) if $plugin =~ /^spamassassin/;
if ( $plugin eq 'sender_permitted_from' ) { if ($plugin eq 'sender_permitted_from') {
$message = 'pass' if $message =~ /^pass/; $message = 'pass' if $message =~ /^pass/;
$message = 'fail' if $message =~ /^fail/; $message = 'fail' if $message =~ /^fail/;
$message = 'skip' if $message =~ /^none/; $message = 'skip' if $message =~ /^none/;
} }
elsif ( $plugin eq 'queue::qmail_2dqueue' ) { elsif ($plugin eq 'queue::qmail_2dqueue') {
($pid) = $message =~ /\(for ([\d]+)\)/; ($pid) = $message =~ /\(for ([\d]+)\)/;
$message = 'pass' if $message =~ /Queuing/; $message = 'pass' if $message =~ /Queuing/;
} }
elsif ( $plugin =~ /(?:early|karma|helo|rcpt_ok)/ ) { elsif ($plugin =~ /(?:early|karma|helo|rcpt_ok)/) {
$message = 'pass' if $message =~ /^pass/; $message = 'pass' if $message =~ /^pass/;
} }
elsif ( $plugin =~ /resolvable_fromhost/ ) { elsif ($plugin =~ /resolvable_fromhost/) {
$message = 'pass' if $message =~ /^pass/; $message = 'pass' if $message =~ /^pass/;
}; }
return ( 'plugin', $pid, $hook, $plugin, $message ); return ('plugin', $pid, $hook, $plugin, $message);
}; }
sub parse_line_plugin_dspam { sub parse_line_plugin_dspam {
my $line = shift; my $line = shift;
my ($tai, $pid, $hook, $plugin, $message ) = split /\s/, $line, 5; my ($tai, $pid, $hook, $plugin, $message) = split /\s/, $line, 5;
$plugin =~ s/:$//; $plugin =~ s/:$//;
if ( $message =~ /Innocent, (\d\.\d\d c)/ ) { if ($message =~ /Innocent, (\d\.\d\d c)/) {
$message = "pass, $1"; $message = "pass, $1";
}; }
if ( $message =~ /Spam, (\d\.\d\d c)/ ) { if ($message =~ /Spam, (\d\.\d\d c)/) {
$message = "fail, $1"; $message = "fail, $1";
}; }
return ( 'plugin', $pid, $hook, $plugin, $message ); return ('plugin', $pid, $hook, $plugin, $message);
}; }
sub parse_line_plugin_spamassassin { sub parse_line_plugin_spamassassin {
my $line = shift; my $line = shift;
my ($tai, $pid, $hook, $plugin, $message ) = split /\s/, $line, 5; my ($tai, $pid, $hook, $plugin, $message) = split /\s/, $line, 5;
$plugin =~ s/:$//; $plugin =~ s/:$//;
if ( $message =~ /pass, Ham, ([\d\-\.]+)\s/ ) { if ($message =~ /pass, Ham, ([\d\-\.]+)\s/) {
$message = "pass, $1"; $message = "pass, $1";
}; }
if ( $message =~ /^fail, Spam,\s([\d\.]+)\s< 100/ ) { if ($message =~ /^fail, Spam,\s([\d\.]+)\s< 100/) {
$message = "fail, $1"; $message = "fail, $1";
}; }
return ( 'plugin', $pid, $hook, $plugin, $message ); return ('plugin', $pid, $hook, $plugin, $message);
}; }
sub parse_line_plugin_p0f { sub parse_line_plugin_p0f {
my $line = shift; my $line = shift;
my ($tai, $pid, $hook, $plugin, $message ) = split /\s/, $line, 5; my ($tai, $pid, $hook, $plugin, $message) = split /\s/, $line, 5;
$plugin =~ s/:$//; $plugin =~ s/:$//;
if ( substr( $message, -5, 5) eq 'hops)' ) { if (substr($message, -5, 5) eq 'hops)') {
($message) = split( /\s\(/, $message ); ($message) = split(/\s\(/, $message);
}; }
$message = 'iOS' if $message =~ /^iOS/; $message = 'iOS' if $message =~ /^iOS/;
$message = 'Solaris' if $message =~ /^Solaris/; $message = 'Solaris' if $message =~ /^Solaris/;
@ -478,68 +504,68 @@ sub parse_line_plugin_p0f {
$message = 'Cisco' if $message =~ /^Cisco/i; $message = 'Cisco' if $message =~ /^Cisco/i;
$message = 'Netware' if $message =~ /Netware/i; $message = 'Netware' if $message =~ /Netware/i;
return ( 'plugin', $pid, $hook, $plugin, $message ); return ('plugin', $pid, $hook, $plugin, $message);
}; }
sub parse_line_cleanup { sub parse_line_cleanup {
my ($line) = @_; my ($line) = @_;
# @tai 85931 cleaning up after 3210 # @tai 85931 cleaning up after 3210
my $pid = (split /\s+/, $line)[-1]; my $pid = (split /\s+/, $line)[-1];
$has_cleanup++; $has_cleanup++;
return ( 'cleanup', $pid, undef, undef, $line ); return ('cleanup', $pid, undef, undef, $line);
}; }
sub get_score { sub get_score {
my $mess = shift; my $mess = shift;
return 3 if $mess eq 'TLS setup returning'; return 3 if $mess eq 'TLS setup returning';
return 3 if $mess =~ /^pass/; return 3 if $mess =~ /^pass/;
return -3 if $mess =~ /^fail/; return -3 if $mess =~ /^fail/;
return -2 if $mess =~ /^negative/; return -2 if $mess =~ /^negative/;
return 2 if $mess =~ /^positive/; return 2 if $mess =~ /^positive/;
return 1 if $mess =~ /^skip/; return 1 if $mess =~ /^skip/;
return 0; return 0;
}; }
sub get_db { sub get_db {
my $db = DBIx::Simple->connect( $dsn, $user, $pass ) my $db = DBIx::Simple->connect($dsn, $user, $pass)
or die DBIx::Simple->error; or die DBIx::Simple->error;
return $db; return $db;
}; }
sub exec_query { sub exec_query {
my $query = shift; my $query = shift;
my $params = shift; my $params = shift;
die "invalid arguments to exec_query!" if @_; die "invalid arguments to exec_query!" if @_;
my @params; my @params;
if ( defined $params ) { if (defined $params) {
@params = ref $params eq 'ARRAY' ? @$params : $params; @params = ref $params eq 'ARRAY' ? @$params : $params;
}; }
my $err = "query failed: $query\n"; my $err = "query failed: $query\n";
if ( scalar @params ) { if (scalar @params) {
$err .= join(',', @params); $err .= join(',', @params);
}; }
#warn "err: $err\n"; #warn "err: $err\n";
if ( $query =~ /INSERT INTO/ ) { if ($query =~ /INSERT INTO/) {
my ( $table ) = $query =~ /INSERT INTO (\w+)\s/; my ($table) = $query =~ /INSERT INTO (\w+)\s/;
$db->query( $query, @params ); $db->query($query, @params);
die "$db->error\n$err" if $db->error ne 'DBI error: '; die "$db->error\n$err" if $db->error ne 'DBI error: ';
my $id = $db->last_insert_id(undef,undef,$table,undef) or die $err; my $id = $db->last_insert_id(undef, undef, $table, undef) or die $err;
return $id; return $id;
} }
elsif ( $query =~ /^UPDATE/i ) { elsif ($query =~ /^UPDATE/i) {
return $db->query( $query, @params ); return $db->query($query, @params);
} }
elsif ( $query =~ /DELETE/ ) { elsif ($query =~ /DELETE/) {
$db->query( $query, @params ) or die $err; $db->query($query, @params) or die $err;
return $db->query("SELECT ROW_COUNT()")->list; return $db->query("SELECT ROW_COUNT()")->list;
}; }
my $r = $db->query( $query, @params )->hashes or die $err; my $r = $db->query($query, @params)->hashes or die $err;
return $r; return $r;
}; }

72
log/show_message
View File

@ -5,68 +5,68 @@ use warnings;
use Data::Dumper; use Data::Dumper;
my $QPDIR = get_qp_dir(); my $QPDIR = get_qp_dir();
my $logfile = "$QPDIR/log/main/current"; my $logfile = "$QPDIR/log/main/current";
my $is_ip = 0; my $is_ip = 0;
my $search = $ARGV[0]; my $search = $ARGV[0];
if ( ! $search ) { if (!$search) {
die "\nusage: $0 [ ip_address | PID ]\n\n"; die "\nusage: $0 [ ip_address | PID ]\n\n";
}; }
if ($search =~ /^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$/) {
if ( $search =~ /^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$/ ) {
#print "it's an IP\n"; #print "it's an IP\n";
$is_ip++; $is_ip++;
}; }
open my $LOG, '<', $logfile or die "unable to open $logfile\n"; open my $LOG, '<', $logfile or die "unable to open $logfile\n";
if ( $is_ip ) { # look for the connection start message for the IP if ($is_ip) { # look for the connection start message for the IP
my $ip_matches; my $ip_matches;
while ( defined (my $line = <$LOG>) ) { while (defined(my $line = <$LOG>)) {
next if ! $line; next if !$line;
my ( $tai, $pid, $mess ) = split /\s/, $line, 3; my ($tai, $pid, $mess) = split /\s/, $line, 3;
if ( 'Connection from ' eq substr( $mess, 0, 16 ) ) { if ('Connection from ' eq substr($mess, 0, 16)) {
my ( $ip ) = (split /\s+/, $mess)[-1]; # IP is last word my ($ip) = (split /\s+/, $mess)[-1]; # IP is last word
$ip = substr $ip, 1, -1; # trim off brackets $ip = substr $ip, 1, -1; # trim off brackets
if ( $ip eq $search ) { if ($ip eq $search) {
$ip_matches++; $ip_matches++;
$search = $pid; $search = $pid;
$is_ip = 0; $is_ip = 0;
}; }
}; }
}; }
seek $LOG, 0, 0; seek $LOG, 0, 0;
die "no pid found for ip $search\n" if $is_ip; die "no pid found for ip $search\n" if $is_ip;
print "showing the last of $ip_matches connnections from $ARGV[0]\n"; print "showing the last of $ip_matches connnections from $ARGV[0]\n";
}; }
print "showing QP message PID $search\n"; print "showing QP message PID $search\n";
while ( defined (my $line = <$LOG>) ) { while (defined(my $line = <$LOG>)) {
next if ! $line; next if !$line;
my ( $tai, $pid, $mess ) = split /\s/, $line, 3; my ($tai, $pid, $mess) = split /\s/, $line, 3;
next if ! $pid; next if !$pid;
print $mess if ( $pid eq $search ); print $mess if ($pid eq $search);
}; }
close $LOG; close $LOG;
sub get_qp_dir { sub get_qp_dir {
foreach my $user ( qw/ qpsmtpd smtpd / ) { foreach my $user (qw/ qpsmtpd smtpd /) {
my ($homedir) = (getpwnam( $user ))[7] or next; my ($homedir) = (getpwnam($user))[7] or next;
if ( -d "$homedir/plugins" ) { if (-d "$homedir/plugins") {
return "$homedir"; return "$homedir";
}; }
foreach my $s ( qw/ smtpd qpsmtpd qpsmtpd-dev / ) { foreach my $s (qw/ smtpd qpsmtpd qpsmtpd-dev /) {
if ( -d "$homedir/$s/plugins" ) { if (-d "$homedir/$s/plugins") {
return "$homedir/$s"; return "$homedir/$s";
}; }
}; }
}; }
if ( -d "./plugins" ) { if (-d "./plugins") {
return Cwd::getcwd(); return Cwd::getcwd();
}; }
}; }

377
log/summarize
View File

@ -15,210 +15,238 @@ my %hide_plugins = map { $_ => 1 } qw/ hostname /;
my $qpdir = get_qp_dir(); my $qpdir = get_qp_dir();
my $file = "$qpdir/log/main/current"; my $file = "$qpdir/log/main/current";
populate_plugins_from_registry(); populate_plugins_from_registry();
my @sorted_plugins = sort { $plugins{$a}{id} <=> $plugins{$b}{id} } keys %plugins; my @sorted_plugins =
sort { $plugins{$a}{id} <=> $plugins{$b}{id} } keys %plugins;
my $fh = File::Tail->new(name=>$file, interval=>1, maxinterval=>1, debug =>1, tail =>1000 ); my $fh = File::Tail->new(
name => $file,
interval => 1,
maxinterval => 1,
debug => 1,
tail => 1000
);
my $printed = 0; my $printed = 0;
my $has_cleanup; my $has_cleanup;
my %formats = ( my %formats = (
ip => "%-15.15s", ip => "%-15.15s",
hostname => "%-20.20s", hostname => "%-20.20s",
distance => "%5.5s", distance => "%5.5s",
'ident::geoip' => "%-20.20s", 'ident::geoip' => "%-20.20s",
'ident::p0f' => "%-10.10s", 'ident::p0f' => "%-10.10s",
count_unrecognized_commands => "%-5.5s", count_unrecognized_commands => "%-5.5s",
unrecognized_commands => "%-5.5s", unrecognized_commands => "%-5.5s",
dnsbl => "%-3.3s", dnsbl => "%-3.3s",
rhsbl => "%-3.3s", rhsbl => "%-3.3s",
relay => "%-3.3s", relay => "%-3.3s",
karma => "%-3.3s", karma => "%-3.3s",
fcrdns => "%-3.3s", fcrdns => "%-3.3s",
earlytalker => "%-3.3s", earlytalker => "%-3.3s",
check_earlytalker => "%-3.3s", check_earlytalker => "%-3.3s",
helo => "%-3.3s", helo => "%-3.3s",
tls => "%-3.3s", tls => "%-3.3s",
'auth::auth_vpopmail' => "%-3.3s", 'auth::auth_vpopmail' => "%-3.3s",
'auth::auth_vpopmaild' => "%-3.3s", 'auth::auth_vpopmaild' => "%-3.3s",
'auth::auth_vpopmail_sql' => "%-3.3s", 'auth::auth_vpopmail_sql' => "%-3.3s",
'auth::auth_checkpassword' => "%-3.3s", 'auth::auth_checkpassword' => "%-3.3s",
badmailfrom => "%-3.3s", badmailfrom => "%-3.3s",
check_badmailfrom => "%-3.3s", check_badmailfrom => "%-3.3s",
sender_permitted_from => "%-3.3s", sender_permitted_from => "%-3.3s",
resolvable_fromhost => "%-3.3s", resolvable_fromhost => "%-3.3s",
'queue::qmail-queue' => "%-3.3s", 'queue::qmail-queue' => "%-3.3s",
connection_time => "%-4.4s", connection_time => "%-4.4s",
); );
my %formats3 = ( my %formats3 = (
%formats, %formats,
badrcptto => "%-3.3s", badrcptto => "%-3.3s",
check_badrcptto => "%-3.3s", check_badrcptto => "%-3.3s",
qmail_deliverable => "%-3.3s", qmail_deliverable => "%-3.3s",
rcpt_ok => "%-3.3s", rcpt_ok => "%-3.3s",
check_basicheaders => "%-3.3s", check_basicheaders => "%-3.3s",
headers => "%-3.3s", headers => "%-3.3s",
uribl => "%-3.3s", uribl => "%-3.3s",
bogus_bounce => "%-3.3s", bogus_bounce => "%-3.3s",
check_bogus_bounce => "%-3.3s", check_bogus_bounce => "%-3.3s",
domainkeys => "%-3.3s", domainkeys => "%-3.3s",
dkim => "%-3.3s", dkim => "%-3.3s",
dmarc => "%-3.3s", dmarc => "%-3.3s",
spamassassin => "%-3.3s", spamassassin => "%-3.3s",
dspam => "%-3.3s", dspam => "%-3.3s",
'virus::clamdscan' => "%-3.3s", 'virus::clamdscan' => "%-3.3s",
); );
while (defined(my $line = $fh->read)) {
while ( defined (my $line = $fh->read) ) {
chomp $line; chomp $line;
next if ! $line; next if !$line;
my ( $type, $pid, $hook, $plugin, $message ) = parse_line( $line ); my ($type, $pid, $hook, $plugin, $message) = parse_line($line);
next if ! $type; next if !$type;
next if $type =~ /^(info|unknown|response|tcpserver)$/; next if $type =~ /^(info|unknown|response|tcpserver)$/;
next if $type eq 'init'; # doesn't occur in all deployment models next if $type eq 'init'; # doesn't occur in all deployment models
if ( ! $pids{$pid} ) { # haven't seen this pid if (!$pids{$pid}) { # haven't seen this pid
next if $type ne 'connect'; # ignore unless connect next if $type ne 'connect'; # ignore unless connect
my ($host, $ip) = split /\s/, $message; my ($host, $ip) = split /\s/, $message;
$ip = substr $ip, 1, -1; $ip = substr $ip, 1, -1;
foreach ( keys %seen_plugins, qw/ helo_host from to / ) { $pids{$pid}{$_} = ''; }; foreach (keys %seen_plugins, qw/ helo_host from to /) {
$pids{$pid}{ip} = $ip; $pids{$pid}{$_} = '';
}
$pids{$pid}{ip} = $ip;
$pids{$pid}{hostname} = $host if $host ne 'Unknown'; $pids{$pid}{hostname} = $host if $host ne 'Unknown';
}; }
if ( $type eq 'close' ) { if ($type eq 'close') {
next if $has_cleanup; # it'll get handled later next if $has_cleanup; # it'll get handled later
print_auto_format($pid, $line); print_auto_format($pid, $line);
delete $pids{$pid}; delete $pids{$pid};
} }
elsif ( $type eq 'cleanup' ) { elsif ($type eq 'cleanup') {
print_auto_format($pid, $line); print_auto_format($pid, $line);
delete $pids{$pid}; delete $pids{$pid};
} }
elsif ( $type eq 'plugin' ) { elsif ($type eq 'plugin') {
next if $plugin eq 'naughty'; # housekeeping only next if $plugin eq 'naughty'; # housekeeping only
if ( ! $pids{$pid}{$plugin} ) { # first entry for this plugin if (!$pids{$pid}{$plugin}) { # first entry for this plugin
$pids{$pid}{$plugin} = $message; $pids{$pid}{$plugin} = $message;
} }
else { # subsequent log entry for this plugin else { # subsequent log entry for this plugin
if ( $pids{$pid}{$plugin} !~ /^(?:pass|fail|skip)/i ) { if ($pids{$pid}{$plugin} !~ /^(?:pass|fail|skip)/i) {
$pids{$pid}{$plugin} = $message; # overwrite 1st $pids{$pid}{$plugin} = $message; # overwrite 1st
} }
else { else {
#print "ignoring subsequent hit on $plugin: $message\n"; #print "ignoring subsequent hit on $plugin: $message\n";
}; }
}; }
if ( $plugin eq 'ident::geoip' ) { if ($plugin eq 'ident::geoip') {
if ( length $message < 3 ) { if (length $message < 3) {
$formats{'ident::geoip'} = "%-3.3s"; $formats{'ident::geoip'} = "%-3.3s";
$formats3{'ident::geoip'} = "%-3.3s"; $formats3{'ident::geoip'} = "%-3.3s";
} }
else { else {
my ($gip, $distance) = $message =~ /(.*?),\s+([\d]+)\skm/; my ($gip, $distance) = $message =~ /(.*?),\s+([\d]+)\skm/;
if ( $distance ) { if ($distance) {
$pids{$pid}{$plugin} = $gip; $pids{$pid}{$plugin} = $gip;
$pids{$pid}{distance} = $distance; $pids{$pid}{distance} = $distance;
}; }
}; }
}; }
} }
elsif ( $type eq 'reject' ) { } elsif ($type eq 'reject') { }
elsif ( $type eq 'connect' ) { } elsif ($type eq 'connect') { }
elsif ( $type eq 'dispatch' ) { elsif ($type eq 'dispatch') {
if ( $message =~ /^dispatching MAIL FROM/i ) { if ($message =~ /^dispatching MAIL FROM/i) {
my ($from) = $message =~ /<(.*?)>/; my ($from) = $message =~ /<(.*?)>/;
$pids{$pid}{from} = $from; $pids{$pid}{from} = $from;
} }
elsif ( $message =~ /^dispatching RCPT TO/i ) { elsif ($message =~ /^dispatching RCPT TO/i) {
my ($to) = $message =~ /<(.*?)>/; my ($to) = $message =~ /<(.*?)>/;
$pids{$pid}{to} = $to; $pids{$pid}{to} = $to;
} }
elsif ( $message =~ m/dispatching (EHLO|HELO) (.*)/ ) { elsif ($message =~ m/dispatching (EHLO|HELO) (.*)/) {
$pids{$pid}{helo_host} = $2; $pids{$pid}{helo_host} = $2;
} }
elsif ( $message eq 'dispatching DATA' ) { } elsif ($message eq 'dispatching DATA') { }
elsif ( $message eq 'dispatching QUIT' ) { } elsif ($message eq 'dispatching QUIT') { }
elsif ( $message eq 'dispatching STARTTLS' ) { } elsif ($message eq 'dispatching STARTTLS') { }
elsif ( $message eq 'dispatching RSET' ) { elsif ($message eq 'dispatching RSET') {
print_auto_format($pid, $line); print_auto_format($pid, $line);
} }
else { else {
# anything here is likely an unrecognized command # anything here is likely an unrecognized command
#print "$message\n"; #print "$message\n";
}; }
} }
else { else {
print "$type $pid $hook $plugin $message\n"; print "$type $pid $hook $plugin $message\n";
}; }
}; }
sub parse_line { sub parse_line {
my $line = shift; my $line = shift;
my ($tai, $pid, $message) = split /\s+/, $line, 3; my ($tai, $pid, $message) = split /\s+/, $line, 3;
return if ! $message; # garbage in the log file return if !$message; # garbage in the log file
# lines seen many times per connection # lines seen many times per connection
return parse_line_plugin( $line ) if substr($message, 0, 1) eq '('; return parse_line_plugin($line) if substr($message, 0, 1) eq '(';
return ( 'dispatch', $pid, undef, undef, $message ) if substr($message, 0, 12) eq 'dispatching '; return ('dispatch', $pid, undef, undef, $message)
return ( 'response', $pid, undef, undef, $message ) if $message =~ /^[2|3]\d\d/; if substr($message, 0, 12) eq 'dispatching ';
return ( 'tcpserver', $pid, undef, undef, undef ) if substr($pid, 0, 10) eq 'tcpserver:'; return ('response', $pid, undef, undef, $message)
if $message =~ /^[2|3]\d\d/;
return ('tcpserver', $pid, undef, undef, undef)
if substr($pid, 0, 10) eq 'tcpserver:';
# lines seen about once per connection # lines seen about once per connection
return ( 'init', $pid, undef, undef, $message ) if substr($message, 0, 19) eq 'Accepted connection'; return ('init', $pid, undef, undef, $message)
return ( 'connect', $pid, undef, undef, substr( $message, 16) ) if substr($message, 0, 15) eq 'Connection from'; if substr($message, 0, 19) eq 'Accepted connection';
return ( 'close', $pid, undef, undef, $message ) if substr($message, 0, 6) eq 'close '; return ('connect', $pid, undef, undef, substr($message, 16))
return ( 'close', $pid, undef, undef, $message ) if substr($message, 0, 20) eq 'click, disconnecting'; if substr($message, 0, 15) eq 'Connection from';
return parse_line_cleanup( $line ) if substr($message, 0, 11) eq 'cleaning up'; return ('close', $pid, undef, undef, $message)
if substr($message, 0, 6) eq 'close ';
return ('close', $pid, undef, undef, $message)
if substr($message, 0, 20) eq 'click, disconnecting';
return parse_line_cleanup($line)
if substr($message, 0, 11) eq 'cleaning up';
# lines seen less than once per connection # lines seen less than once per connection
return ( 'info', $pid, undef, undef, $message ) if $message eq 'spooling message to disk'; return ('info', $pid, undef, undef, $message)
return ( 'reject', $pid, undef, undef, $message ) if $message =~ /^[4|5]\d\d/; if $message eq 'spooling message to disk';
return ( 'reject', $pid, undef, undef, $message ) if substr($message, 0, 14) eq 'deny mail from'; return ('reject', $pid, undef, undef, $message) if $message =~ /^[4|5]\d\d/;
return ( 'reject', $pid, undef, undef, $message ) if substr($message, 0, 18) eq 'denysoft mail from'; return ('reject', $pid, undef, undef, $message)
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 15) eq 'Lost connection'; if substr($message, 0, 14) eq 'deny mail from';
return ( 'info', $pid, undef, undef, $message ) if $message eq 'auth success cleared naughty'; return ('reject', $pid, undef, undef, $message)
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 15) eq 'Running as user'; if substr($message, 0, 18) eq 'denysoft mail from';
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 16) eq 'Loaded Qpsmtpd::'; return ('info', $pid, undef, undef, $message)
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 24) eq 'Permissions on spool_dir'; if substr($message, 0, 15) eq 'Lost connection';
return ( 'info', $pid, undef, undef, $message ) if substr($message, 0, 13) eq 'Listening on '; return ('info', $pid, undef, undef, $message)
if $message eq 'auth success cleared naughty';
return ('info', $pid, undef, undef, $message)
if substr($message, 0, 15) eq 'Running as user';
return ('info', $pid, undef, undef, $message)
if substr($message, 0, 16) eq 'Loaded Qpsmtpd::';
return ('info', $pid, undef, undef, $message)
if substr($message, 0, 24) eq 'Permissions on spool_dir';
return ('info', $pid, undef, undef, $message)
if substr($message, 0, 13) eq 'Listening on ';
return ( 'err', $pid, undef, undef, $message ) if $line =~ /at [\S]+ line \d/; # generic perl error return ('err', $pid, undef, undef, $message)
if $line =~ /at [\S]+ line \d/; # generic perl error
print "UNKNOWN LINE: $line\n"; print "UNKNOWN LINE: $line\n";
return ( 'unknown', $pid, undef, undef, $message ); return ('unknown', $pid, undef, undef, $message);
}; }
sub parse_line_plugin { sub parse_line_plugin {
my ($line) = @_; my ($line) = @_;
# @tai 13486 (connect) ident::p0f: Windows (XP/2000 (RFC1323+, w, tstamp-)) # @tai 13486 (connect) ident::p0f: Windows (XP/2000 (RFC1323+, w, tstamp-))
# @tai 13681 (connect) dnsbl: fail, NAUGHTY # @tai 13681 (connect) dnsbl: fail, NAUGHTY
# @tai 15787 (connect) karma: pass, no penalty (0 naughty, 3 nice, 3 connects) # @tai 15787 (connect) karma: pass, no penalty (0 naughty, 3 nice, 3 connects)
# @tai 77603 (queue) queue::qmail_2dqueue: (for 77590) Queuing to /var/qmail/bin/qmail-queue # @tai 77603 (queue) queue::qmail_2dqueue: (for 77590) Queuing to /var/qmail/bin/qmail-queue
my ($tai, $pid, $hook, $plugin, $message ) = split /\s/, $line, 5; my ($tai, $pid, $hook, $plugin, $message) = split /\s/, $line, 5;
$plugin =~ s/:$//; $plugin =~ s/:$//;
if ( $plugin =~ /_3a/ ) { if ($plugin =~ /_3a/) {
($plugin) = split /_3a/, $plugin; # trim :N off the plugin log entry ($plugin) = split /_3a/, $plugin; # trim :N off the plugin log entry
}; }
$plugin =~ s/_2d/-/g; $plugin =~ s/_2d/-/g;
$plugin = $plugin_aliases{$plugin} if $plugin_aliases{$plugin}; # map alias to master $plugin = $plugin_aliases{$plugin}
if ( $hook eq '(queue)' ) { if $plugin_aliases{$plugin}; # map alias to master
if ($hook eq '(queue)') {
($pid) = $message =~ /\(for ([\d]+)\)\s/; ($pid) = $message =~ /\(for ([\d]+)\)\s/;
$message = 'pass'; $message = 'pass';
}; }
return ( 'plugin', $pid, $hook, $plugin, $message ); return ('plugin', $pid, $hook, $plugin, $message);
}; }
sub parse_line_cleanup { sub parse_line_cleanup {
my ($line) = @_; my ($line) = @_;
# @tai 85931 cleaning up after 3210 # @tai 85931 cleaning up after 3210
my $pid = (split /\s+/, $line)[-1]; my $pid = (split /\s+/, $line)[-1];
$has_cleanup++; $has_cleanup++;
return ( 'cleanup', $pid, undef, undef, $line ); return ('cleanup', $pid, undef, undef, $line);
}; }
sub print_auto_format { sub print_auto_format {
my ($pid, $line) = @_; my ($pid, $line) = @_;
@ -227,52 +255,53 @@ sub print_auto_format {
my @headers; my @headers;
my @values; my @values;
foreach my $plugin ( qw/ ip hostname distance /, @sorted_plugins ) { foreach my $plugin (qw/ ip hostname distance /, @sorted_plugins) {
if ( defined $pids{$pid}{$plugin} ) { if (defined $pids{$pid}{$plugin}) {
if ( ! $seen_plugins{$plugin} ) { # first time seeing this plugin if (!$seen_plugins{$plugin}) { # first time seeing this plugin
$printed = 0; # force header print $printed = 0; # force header print
}; }
$seen_plugins{$plugin}++; $seen_plugins{$plugin}++;
}; }
next if ! $seen_plugins{$plugin}; # hide unused plugins next if !$seen_plugins{$plugin}; # hide unused plugins
if ( $hide_plugins{$plugin} ) { # user doesn't want to see if ($hide_plugins{$plugin}) { # user doesn't want to see
delete $pids{$pid}{$plugin}; delete $pids{$pid}{$plugin};
next; next;
}; }
if ( defined $pids{$pid}{helo_host} && $plugin =~ /helo/ ) { if (defined $pids{$pid}{helo_host} && $plugin =~ /helo/) {
$format .= " %-18.18s"; $format .= " %-18.18s";
push @values, substr( delete $pids{$pid}{helo_host}, -18, 18); push @values, substr(delete $pids{$pid}{helo_host}, -18, 18);
push @headers, 'HELO'; push @headers, 'HELO';
} }
elsif ( defined $pids{$pid}{from} && $plugin =~ /from/ ) { elsif (defined $pids{$pid}{from} && $plugin =~ /from/) {
$format .= " %-20.20s"; $format .= " %-20.20s";
push @values, substr( delete $pids{$pid}{from}, -20, 20); push @values, substr(delete $pids{$pid}{from}, -20, 20);
push @headers, 'MAIL FROM'; push @headers, 'MAIL FROM';
} }
elsif ( defined $pids{$pid}{to} && $plugin =~ /to|rcpt|recipient/ ) { elsif (defined $pids{$pid}{to} && $plugin =~ /to|rcpt|recipient/) {
$format .= " %-20.20s"; $format .= " %-20.20s";
push @values, delete $pids{$pid}{to}; push @values, delete $pids{$pid}{to};
push @headers, 'RCPT TO'; push @headers, 'RCPT TO';
}; }
$format .= $formats3{$plugin} ? " $formats3{$plugin}" : " %-10.10s"; $format .= $formats3{$plugin} ? " $formats3{$plugin}" : " %-10.10s";
if ( defined $pids{$pid}{$plugin} ) { if (defined $pids{$pid}{$plugin}) {
push @values, show_symbol( delete $pids{$pid}{$plugin} ); push @values, show_symbol(delete $pids{$pid}{$plugin});
} }
else { else {
push @values, ''; push @values, '';
}; }
push @headers, ($plugins{$plugin}{abb3} ? $plugins{$plugin}{abb3} : $plugin); push @headers,
($plugins{$plugin}{abb3} ? $plugins{$plugin}{abb3} : $plugin);
} }
$format .= "\n"; $format .= "\n";
printf( "\n$format", @headers ) if ( ! $printed || $printed % 20 == 0 ); printf("\n$format", @headers) if (!$printed || $printed % 20 == 0);
printf( $format, @values ); printf($format, @values);
print Data::Dumper::Dumper( $pids{$pid} ) if keys %{$pids{$pid}}; print Data::Dumper::Dumper($pids{$pid}) if keys %{$pids{$pid}};
$printed++; $printed++;
}; }
sub show_symbol { sub show_symbol {
my $mess = shift; my $mess = shift;
@ -288,46 +317,46 @@ sub show_symbol {
return ' !' if $mess =~ /^error[,:\s]/i; return ' !' if $mess =~ /^error[,:\s]/i;
$mess =~ s/\s\s/ /g; $mess =~ s/\s\s/ /g;
return $mess; return $mess;
}; }
sub get_qp_dir { sub get_qp_dir {
foreach my $user ( qw/ qpsmtpd smtpd / ) { foreach my $user (qw/ qpsmtpd smtpd /) {
my ($homedir) = (getpwnam( $user ))[7] or next; my ($homedir) = (getpwnam($user))[7] or next;
if ( -d "$homedir/plugins" ) { if (-d "$homedir/plugins") {
return "$homedir"; return "$homedir";
}; }
foreach my $s ( qw/ smtpd qpsmtpd qpsmtpd-dev / ) { foreach my $s (qw/ smtpd qpsmtpd qpsmtpd-dev /) {
if ( -d "$homedir/$s/plugins" ) { if (-d "$homedir/$s/plugins") {
return "$homedir/$s"; return "$homedir/$s";
}; }
}; }
}; }
if ( -d "./plugins" ) { if (-d "./plugins") {
return Cwd::getcwd(); return Cwd::getcwd();
}; }
}; }
sub populate_plugins_from_registry { sub populate_plugins_from_registry {
my $file = "$qpdir/plugins/registry.txt"; my $file = "$qpdir/plugins/registry.txt";
if ( ! -f $file ) { if (!-f $file) {
die "unable to find plugin registry\n"; die "unable to find plugin registry\n";
}; }
open my $F, '<', $file; open my $F, '<', $file;
while ( defined ( my $line = <$F> ) ) { while (defined(my $line = <$F>)) {
next if $line =~ /^#/; # discard comments next if $line =~ /^#/; # discard comments
my ($id, $name, $abb3, $abb5, $aliases) = split /\s+/, $line; my ($id, $name, $abb3, $abb5, $aliases) = split /\s+/, $line;
next if ! defined $name; next if !defined $name;
$plugins{$name} = { id=>$id, abb3=>$abb3, abb5=>$abb5 }; $plugins{$name} = {id => $id, abb3 => $abb3, abb5 => $abb5};
next if ! $aliases; next if !$aliases;
$aliases =~ s/\s+//g; $aliases =~ s/\s+//g;
$plugins{$name}{aliases} = $aliases; $plugins{$name}{aliases} = $aliases;
foreach my $a ( split /,/, $aliases ) { foreach my $a (split /,/, $aliases) {
$plugin_aliases{$a} = $name; $plugin_aliases{$a} = $name;
}; }
}; }
}; }

40
log/watch
View File

@ -3,7 +3,7 @@
use strict; use strict;
use warnings; use warnings;
$|++; # OUTPUT_AUTOFLUSH $|++; # OUTPUT_AUTOFLUSH
use Cwd; use Cwd;
use Data::Dumper; use Data::Dumper;
@ -11,28 +11,34 @@ use File::Tail;
my $dir = get_qp_dir() or die "unable to find QP home dir"; my $dir = get_qp_dir() or die "unable to find QP home dir";
my $file = "$dir/log/main/current"; my $file = "$dir/log/main/current";
my $fh = File::Tail->new(name=>$file, interval=>1, maxinterval=>1, debug =>1, tail =>300 ); my $fh = File::Tail->new(
name => $file,
interval => 1,
maxinterval => 1,
debug => 1,
tail => 300
);
while ( defined (my $line = $fh->read) ) { while (defined(my $line = $fh->read)) {
my (undef, $line) = split /\s/, $line, 2; # strip off tai timestamps my (undef, $line) = split /\s/, $line, 2; # strip off tai timestamps
print $line; print $line;
}; }
sub get_qp_dir { sub get_qp_dir {
foreach my $user ( qw/ qpsmtpd smtpd / ) { foreach my $user (qw/ qpsmtpd smtpd /) {
my ($homedir) = (getpwnam( $user ))[7] or next; my ($homedir) = (getpwnam($user))[7] or next;
if ( -d "$homedir/plugins" ) { if (-d "$homedir/plugins") {
return "$homedir"; return "$homedir";
}; }
foreach my $s ( qw/ smtpd qpsmtpd qpsmtpd-dev / ) { foreach my $s (qw/ smtpd qpsmtpd qpsmtpd-dev /) {
if ( -d "$homedir/$s/plugins" ) { if (-d "$homedir/$s/plugins") {
return "$homedir/$s"; return "$homedir/$s";
}; }
}; }
}; }
if ( -d "./plugins" ) { if (-d "./plugins") {
return Cwd::getcwd(); return Cwd::getcwd();
}; }
}; }