byterazor/qpsmtpd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

log2sql: populate plugins table from registry.txt

Browse Source 
much easier for local customizations.
moved SQL connection settings to config/log2sql
This commit is contained in:
Matt Simerson 2013-04-22 02:12:53 -04:00
parent 8b95e9053d
commit 78ac01df76
4 changed files with 113 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config.sample/log2sql Normal file
View File

@ -0,0 +1,4 @@
# comments are allowed
dsn = DBI:mysql:database=qpsmtpd;host=db;port=3306
user = qplog
pass = can mysql have 6 spaces in a passphrase?

74
log/log2sql
View File

@ -6,21 +6,19 @@ use warnings;
use Cwd; use Cwd;
use Data::Dumper; use Data::Dumper;
use DBIx::Simple; use DBIx::Simple;
use IO::File;
use File::stat; use File::stat;
use Time::TAI64 qw/ tai2unix /; use Time::TAI64 qw/ tai2unix /;
$Data::Dumper::Sortkeys = 1; $Data::Dumper::Sortkeys = 1;
my $dsn = 'DBI:mysql:database=qpsmtpd;host=db;port=3306';
my $user = 'qplog';
my $pass = 't0ps3cret';
my $logdir = get_log_dir(); my $logdir = get_log_dir();
my @logfiles = get_logfiles($logdir); my @logfiles = get_logfiles($logdir);
my (%plugins, %os, %message_ids); my (%plugins, %os, %message_ids);
my $has_cleanup; my $has_cleanup;
my $db = get_db(); my $db = get_db();
check_plugins_table();
foreach my $file (@logfiles) { foreach my $file (@logfiles) {
my ($fid, $offset) = check_logfile($file); my ($fid, $offset) = check_logfile($file);
@ -208,6 +206,7 @@ sub parse_logfile {
#warn "type: $type\n"; #warn "type: $type\n";
if ($type eq 'plugin') { if ($type eq 'plugin') {
next if $plugin eq 'naughty'; # housekeeping only next if $plugin eq 'naughty'; # housekeeping only
next if $plugin eq 'karma' && 'karma adjust' eq substr($message,0,12);
insert_plugin($msg_id, $plugin, $message); insert_plugin($msg_id, $plugin, $message);
} }
elsif ($type eq 'queue') { elsif ($type eq 'queue') {
@ -529,12 +528,70 @@ sub get_score {
sub get_db { sub get_db {
my $db = DBIx::Simple->connect($dsn, $user, $pass) my %dbv = get_config('log2sql');
$dbv{dsn} ||= 'DBI:mysql:database=qpsmtpd;host=db;port=3306';
$dbv{user} ||= 'qplog';
$dbv{pass} ||= 't0ps3cret';
print Dumper(\%dbv);
my $db = DBIx::Simple->connect($dbv{dsn}, $dbv{user}, $dbv{pass})
or die DBIx::Simple->error; or die DBIx::Simple->error;
return $db; return $db;
} }
sub get_config {
my $file = shift or die "missing file name\n";
my %values;
foreach my $line ( get_config_contents( $file ) ) {
next if $line =~ /^#/;
chomp $line;
my ($key,$val) = split /\s*=\s*/, $line, 2;
$values{$key} = $val;
};
return %values;
};
sub get_config_contents {
my $name = shift;
my @config_dirs = qw[ config ../config log plugins ];
foreach my $dir ( @config_dirs ) {
next if ! -f "$dir/$name";
my $fh = IO::File->new();
if ( ! $fh->open( "$dir/$name", '<' ) ) {
warn "unable to open config file $dir/$name\n";
next;
};
my @contents = <$fh>;
return @contents;
};
};
sub check_plugins_table {
my $rows = exec_query( 'SELECT COUNT(*) FROM plugin');
return if scalar @$rows != 0;
my @lines = get_config_contents('registry.txt');
foreach my $line ( @lines ) {
next if $line =~ /^\s*#/; # ignore comments
chomp $line;
next if ! $line;
my ($id, $name, $abb3, $abb5, $aliases) = split /\s+/, $line, 5;
my $q = "REPLACE INTO plugin (id,name,abb3,abb5) VALUES (??)";
print "query: $q, $id, $name, $abb3, $abb5\n";
exec_query($q, [$id, $name, $abb3, $abb5 ]);
next if ! $aliases;
foreach my $alias ( split /\s*,\s*/, $aliases ) {
next if ! $alias;
my $aq = "REPLACE INTO plugin_aliases (plugin_id,name) VALUES (??)";
print "aqury: $aq, $id, $alias\n";
exec_query($aq, [$id, $alias]);
};
};
};
sub exec_query { sub exec_query {
my $query = shift; my $query = shift;
my $params = shift; my $params = shift;
@ -550,10 +607,11 @@ sub exec_query {
} }
#warn "err: $err\n"; #warn "err: $err\n";
if ($query =~ /INSERT INTO/) { if ($query =~ /(?:REPLACE|INSERT) INTO/) {
my ($table) = $query =~ /INSERT INTO (\w+)\s/; my ($table) = $query =~ /(?:REPLACE|INSERT) INTO (\w+)\s/;
$db->query($query, @params); $db->query($query, @params);
die "$db->error\n$err" if $db->error ne 'DBI error: '; warn "$db->error\n$err" if $db->error ne 'DBI error: ';
return if $query =~ /^REPLACE/;
my $id = $db->last_insert_id(undef, undef, $table, undef) or die $err; my $id = $db->last_insert_id(undef, undef, $table, undef) or die $err;
return $id; return $id;
} }

146
log/log2sql.sql
View File

@ -13,35 +13,34 @@
DROP TABLE IF EXISTS `log`; DROP TABLE IF EXISTS `log`;
CREATE TABLE `log` ( CREATE TABLE `log` (
`id` int(11) unsigned NOT NULL auto_increment, `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`inode` int(11) unsigned NOT NULL, `inode` int(11) unsigned NOT NULL,
`size` int(11) unsigned NOT NULL, `size` int(11) unsigned NOT NULL,
`name` varchar(30) NOT NULL default '', `name` varchar(30) NOT NULL DEFAULT '',
`created` datetime default NULL, `created` datetime DEFAULT NULL,
PRIMARY KEY (`id`) PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8; ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
# Dump of table message # Dump of table message
# ------------------------------------------------------------ # ------------------------------------------------------------
DROP TABLE IF EXISTS `message`; DROP TABLE IF EXISTS `message`;
CREATE TABLE `message` ( CREATE TABLE `message` (
`id` int(11) unsigned NOT NULL auto_increment, `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`file_id` int(10) unsigned NOT NULL, `file_id` int(10) unsigned NOT NULL,
`connect_start` datetime NOT NULL, `connect_start` datetime NOT NULL,
`ip` int(10) unsigned NOT NULL, `ip` int(10) unsigned NOT NULL,
`qp_pid` int(10) unsigned NOT NULL, `qp_pid` int(10) unsigned NOT NULL,
`result` tinyint(3) NOT NULL default '0', `result` tinyint(3) NOT NULL DEFAULT '0',
`distance` mediumint(8) unsigned default NULL, `distance` mediumint(8) unsigned DEFAULT NULL,
`time` decimal(3,2) unsigned default NULL, `time` decimal(3,2) unsigned DEFAULT NULL,
`os_id` tinyint(3) unsigned default NULL, `os_id` tinyint(3) unsigned DEFAULT NULL,
`hostname` varchar(128) default NULL, `hostname` varchar(128) DEFAULT NULL,
`helo` varchar(128) default NULL, `helo` varchar(128) DEFAULT NULL,
`mail_from` varchar(128) default NULL, `mail_from` varchar(128) DEFAULT NULL,
`rcpt_to` varchar(128) default NULL, `rcpt_to` varchar(128) DEFAULT NULL,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
KEY `file_id` (`file_id`), KEY `file_id` (`file_id`),
CONSTRAINT `message_ibfk_1` FOREIGN KEY (`file_id`) REFERENCES `log` (`id`) ON DELETE CASCADE ON UPDATE CASCADE CONSTRAINT `message_ibfk_1` FOREIGN KEY (`file_id`) REFERENCES `log` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8; ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@ -54,12 +53,12 @@ CREATE TABLE `message` (
DROP TABLE IF EXISTS `message_plugin`; DROP TABLE IF EXISTS `message_plugin`;
CREATE TABLE `message_plugin` ( CREATE TABLE `message_plugin` (
`id` int(11) unsigned NOT NULL auto_increment, `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`msg_id` int(11) unsigned NOT NULL, `msg_id` int(11) unsigned NOT NULL,
`plugin_id` int(4) unsigned NOT NULL, `plugin_id` int(4) unsigned NOT NULL,
`result` tinyint(4) NOT NULL, `result` tinyint(4) NOT NULL,
`string` varchar(128) default NULL, `string` varchar(128) DEFAULT NULL,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
KEY `msg_id` (`msg_id`), KEY `msg_id` (`msg_id`),
KEY `plugin_id` (`plugin_id`), KEY `plugin_id` (`plugin_id`),
CONSTRAINT `message_plugin_ibfk_1` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT `message_plugin_ibfk_1` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
@ -67,16 +66,15 @@ CREATE TABLE `message_plugin` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8; ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
# Dump of table os # Dump of table os
# ------------------------------------------------------------ # ------------------------------------------------------------
DROP TABLE IF EXISTS `os`; DROP TABLE IF EXISTS `os`;
CREATE TABLE `os` ( CREATE TABLE `os` (
`id` tinyint(3) unsigned NOT NULL auto_increment, `id` tinyint(3) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(36) default NULL, `name` varchar(36) DEFAULT NULL,
PRIMARY KEY (`id`) PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8; ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
LOCK TABLES `os` WRITE; LOCK TABLES `os` WRITE;
@ -114,81 +112,14 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `plugin`; DROP TABLE IF EXISTS `plugin`;
CREATE TABLE `plugin` ( CREATE TABLE `plugin` (
`id` int(4) unsigned NOT NULL auto_increment, `id` int(4) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(35) character set utf8 NOT NULL default '', `name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '',
`abb3` char(3) character set utf8 default NULL, `abb3` char(3) CHARACTER SET utf8 DEFAULT NULL,
`abb5` char(5) character set utf8 default NULL, `abb5` char(5) CHARACTER SET utf8 DEFAULT NULL,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `abb3` (`abb3`),
UNIQUE KEY `abb5` (`abb5`) UNIQUE KEY `abb5` (`abb5`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
LOCK TABLES `plugin` WRITE;
/*!40000 ALTER TABLE `plugin` DISABLE KEYS */;
INSERT INTO `plugin` (`id`, `name`, `abb3`, `abb5`)
VALUES
(1,'hosts_allow','alw','allow'),
(2,'ident::geoip','geo','geoip'),
(3,'ident::p0f','p0f',' p0f'),
(5,'karma','krm','karma'),
(6,'dnsbl','dbl','dnsbl'),
(7,'relay','rly','relay'),
(9,'earlytalker','ear','early'),
(15,'helo','hlo','helo'),
(16,'tls','tls',' tls'),
(20,'dont_require_anglebrackets','rab','drabs'),
(21,'unrecognized_commands','cmd','uncmd'),
(22,'noop','nop','noop'),
(23,'random_error','rnd','rande'),
(24,'milter','mtr','mlter'),
(25,'content_log','log','colog'),
(30,'auth::vpopmail_sql','aut','vpsql'),
(31,'auth::vpopmaild','vpd','vpopd'),
(32,'auth::vpopmail','vpo','vpop'),
(33,'auth::checkpasswd','ckp','chkpw'),
(34,'auth::cvs_unix_local','cvs','cvsul'),
(35,'auth::flat_file','flt','aflat'),
(36,'auth::ldap_bind','ldp','aldap'),
(40,'badmailfrom','bmf','badmf'),
(41,'badmailfromto','bmt','bfrto'),
(42,'rhsbl','rbl','rhsbl'),
(44,'resolvable_fromhost','rfh','rsvfh'),
(45,'sender_permitted_from','spf',' spf'),
(50,'badrcptto','bto','badto'),
(51,'rcpt_map','rmp','rcmap'),
(52,'rcpt_regex','rcx','rcrex'),
(53,'qmail_deliverable','qmd',' qmd'),
(55,'rcpt_ok','rok','rcpok'),
(58,'bogus_bounce','bog','bogus'),
(59,'greylisting','gry','greyl'),
(60,'headers','hdr','headr'),
(61,'loop','lop','loop'),
(62,'uribl','uri','uribl'),
(63,'domainkeys','dk','dkey'),
(64,'dkim','dkm','dkim'),
(65,'spamassassin','spm','spama'),
(66,'dspam','dsp','dspam'),
(70,'virus::aveclient','vav','avirs'),
(71,'virus::bitdefender','vbd','bitdf'),
(72,'virus::clamav','cav','clamv'),
(73,'virus::clamdscan','cad','clamd'),
(74,'virus::hbedv','hbv','hbedv'),
(75,'virus::kavscanner','kav','kavsc'),
(76,'virus::klez_filter','klz','vklez'),
(77,'virus::sophie','sop','sophe'),
(78,'virus::uvscan','uvs','uvscn'),
(80,'queue::qmail-queue','qqm','queue'),
(81,'queue::maildir','qdr','qudir'),
(82,'queue::postfix-queue','qpf','qupfx'),
(83,'queue::smtp-forward','qfw','qufwd'),
(84,'queue::exim-bsmtp','qxm','qexim'),
(98,'quit_fortune','for','fortu'),
(99,'connection_time','tim','time');
/*!40000 ALTER TABLE `plugin` ENABLE KEYS */;
UNLOCK TABLES;
# Dump of table plugin_aliases # Dump of table plugin_aliases
# ------------------------------------------------------------ # ------------------------------------------------------------
@ -197,33 +128,10 @@ DROP TABLE IF EXISTS `plugin_aliases`;
CREATE TABLE `plugin_aliases` ( CREATE TABLE `plugin_aliases` (
`plugin_id` int(11) unsigned NOT NULL, `plugin_id` int(11) unsigned NOT NULL,
`name` varchar(35) character set utf8 NOT NULL default '', `name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '',
KEY `plugin_id` (`plugin_id`), UNIQUE KEY `plugin_id` (`plugin_id`,`name`)
CONSTRAINT `plugin_id` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
LOCK TABLES `plugin_aliases` WRITE;
/*!40000 ALTER TABLE `plugin_aliases` DISABLE KEYS */;
INSERT INTO `plugin_aliases` (`plugin_id`, `name`)
VALUES
(60,'check_basicheaders'),
(44,'require_resolvable_fromhost'),
(21,'count_unrecognized_commands'),
(9,'check_earlytalker'),
(40,'check_badmailfrom'),
(50,'check_badrcptto'),
(58,'check_bogus_bounce'),
(15,'check_spamhelo'),
(3,'ident::p0f_3a0'),
(80,'queue::qmail_2dqueue'),
(22,'noop_counter');
/*!40000 ALTER TABLE `plugin_aliases` ENABLE KEYS */;
UNLOCK TABLES;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;

29
plugins/registry.txt
View File

@ -2,12 +2,15 @@
# additional plugins running, add them here. # additional plugins running, add them here.
# Fields are whitespace delimited. Columns are ordered by numeric plugin ID. # Fields are whitespace delimited. Columns are ordered by numeric plugin ID.
# #
# the order of plugins in this file determines the order they appear in
# summary output
#
#id name abb3 abb5 aliases #id name abb3 abb5 aliases
# #
201 hosts_allow alw allow 201 hosts_allow alw allow
202 ident::geoip geo geoip 202 ident::geoip geo geoip
203 ident::p0f p0f p0f 203 ident::p0f p0f p0f ident::p0f_3a0,ident::p0f_3a1
204 ident::p0f_3a0 p0f p0f
205 karma krm karma 205 karma krm karma
206 dnsbl dbl dnsbl 206 dnsbl dbl dnsbl
207 relay rly relay check_relay,check_norelay,relay_only 207 relay rly relay check_relay,check_norelay,relay_only
@ -26,13 +29,13 @@
# #
# Authentication # Authentication
# #
400 auth::auth_vpopmail_sql aut vpsql 400 auth::auth_vpopmail_sql avq avsql
401 auth::auth_vpopmaild vpd vpopd 401 auth::auth_vpopmaild avd vpopd
402 auth::auth_vpopmail vpo vpop 402 auth::auth_vpopmail avp vpop
403 auth::auth_checkpasswd ckp chkpw 403 auth::auth_checkpassword ack chkpw
404 auth::auth_cvs_unix_local cvs cvsul 404 auth::auth_cvs_unix_local acv cvsul
405 auth::auth_flat_file flt aflat 405 auth::auth_flat_file aff aflat
406 auth::auth_ldap_bind ldp aldap 406 auth::auth_ldap_bind ald aldap
407 auth::authdeny dny adeny 407 auth::authdeny dny adeny
# #
# Sender / Envelope From # Sender / Envelope From
@ -80,11 +83,11 @@
# #
# Queue Plugins # Queue Plugins
# #
800 queue::qmail-queue qqm queue 800 queue::qmail-queue qqm queue queue::qmail_2dqueue
801 queue::maildir qdr qudir 801 queue::maildir qdr qudir
802 queue::postfix-queue qpf qupfx 802 queue::postfix-queue qpf qupfx queue::postfix_2dqueue
803 queue::smtp-forward qfw qufwd 803 queue::smtp-forward qfw qufwd queue::smtp_2dqueue
804 queue::exim-bsmtp qxm qexim 804 queue::exim-bsmtp qxm qexim queue::exim_2dbsmtp
900 quit_fortune for fortu 900 quit_fortune for fortu