From 78ac01df760f23e88f70fb44b67c8642b7d2d7e6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 22 Apr 2013 02:12:53 -0400 Subject: [PATCH] log2sql: populate plugins table from registry.txt much easier for local customizations. moved SQL connection settings to config/log2sql --- config.sample/log2sql | 4 ++ log/log2sql | 74 ++++++++++++++++++--- log/log2sql.sql | 146 ++++++++---------------------------------- plugins/registry.txt | 29 +++++---- 4 files changed, 113 insertions(+), 140 deletions(-) create mode 100644 config.sample/log2sql diff --git a/config.sample/log2sql b/config.sample/log2sql new file mode 100644 index 0000000..5b02654 --- /dev/null +++ b/config.sample/log2sql @@ -0,0 +1,4 @@ +# comments are allowed +dsn = DBI:mysql:database=qpsmtpd;host=db;port=3306 +user = qplog +pass = can mysql have 6 spaces in a passphrase? diff --git a/log/log2sql b/log/log2sql index fa8010e..89bb1f1 100755 --- a/log/log2sql +++ b/log/log2sql @@ -6,21 +6,19 @@ use warnings; use Cwd; use Data::Dumper; use DBIx::Simple; +use IO::File; use File::stat; use Time::TAI64 qw/ tai2unix /; $Data::Dumper::Sortkeys = 1; -my $dsn = 'DBI:mysql:database=qpsmtpd;host=db;port=3306'; -my $user = 'qplog'; -my $pass = 't0ps3cret'; - my $logdir = get_log_dir(); my @logfiles = get_logfiles($logdir); my (%plugins, %os, %message_ids); my $has_cleanup; my $db = get_db(); +check_plugins_table(); foreach my $file (@logfiles) { my ($fid, $offset) = check_logfile($file); @@ -208,6 +206,7 @@ sub parse_logfile { #warn "type: $type\n"; if ($type eq 'plugin') { next if $plugin eq 'naughty'; # housekeeping only + next if $plugin eq 'karma' && 'karma adjust' eq substr($message,0,12); insert_plugin($msg_id, $plugin, $message); } elsif ($type eq 'queue') { @@ -529,12 +528,70 @@ sub get_score { sub get_db { - my $db = DBIx::Simple->connect($dsn, $user, $pass) + my %dbv = get_config('log2sql'); + + $dbv{dsn} ||= 'DBI:mysql:database=qpsmtpd;host=db;port=3306'; + $dbv{user} ||= 'qplog'; + $dbv{pass} ||= 't0ps3cret'; + + print Dumper(\%dbv); + my $db = DBIx::Simple->connect($dbv{dsn}, $dbv{user}, $dbv{pass}) or die DBIx::Simple->error; return $db; } +sub get_config { + my $file = shift or die "missing file name\n"; + my %values; + foreach my $line ( get_config_contents( $file ) ) { + next if $line =~ /^#/; + chomp $line; + my ($key,$val) = split /\s*=\s*/, $line, 2; + $values{$key} = $val; + }; + return %values; +}; + +sub get_config_contents { + my $name = shift; + + my @config_dirs = qw[ config ../config log plugins ]; + foreach my $dir ( @config_dirs ) { + next if ! -f "$dir/$name"; + + my $fh = IO::File->new(); + if ( ! $fh->open( "$dir/$name", '<' ) ) { + warn "unable to open config file $dir/$name\n"; + next; + }; + my @contents = <$fh>; + return @contents; + }; +}; + +sub check_plugins_table { + my $rows = exec_query( 'SELECT COUNT(*) FROM plugin'); + return if scalar @$rows != 0; + my @lines = get_config_contents('registry.txt'); + foreach my $line ( @lines ) { + next if $line =~ /^\s*#/; # ignore comments + chomp $line; + next if ! $line; + my ($id, $name, $abb3, $abb5, $aliases) = split /\s+/, $line, 5; + my $q = "REPLACE INTO plugin (id,name,abb3,abb5) VALUES (??)"; + print "query: $q, $id, $name, $abb3, $abb5\n"; + exec_query($q, [$id, $name, $abb3, $abb5 ]); + next if ! $aliases; + foreach my $alias ( split /\s*,\s*/, $aliases ) { + next if ! $alias; + my $aq = "REPLACE INTO plugin_aliases (plugin_id,name) VALUES (??)"; + print "aqury: $aq, $id, $alias\n"; + exec_query($aq, [$id, $alias]); + }; + }; +}; + sub exec_query { my $query = shift; my $params = shift; @@ -550,10 +607,11 @@ sub exec_query { } #warn "err: $err\n"; - if ($query =~ /INSERT INTO/) { - my ($table) = $query =~ /INSERT INTO (\w+)\s/; + if ($query =~ /(?:REPLACE|INSERT) INTO/) { + my ($table) = $query =~ /(?:REPLACE|INSERT) INTO (\w+)\s/; $db->query($query, @params); - die "$db->error\n$err" if $db->error ne 'DBI error: '; + warn "$db->error\n$err" if $db->error ne 'DBI error: '; + return if $query =~ /^REPLACE/; my $id = $db->last_insert_id(undef, undef, $table, undef) or die $err; return $id; } diff --git a/log/log2sql.sql b/log/log2sql.sql index 4f975eb..0c06f35 100644 --- a/log/log2sql.sql +++ b/log/log2sql.sql @@ -13,35 +13,34 @@ DROP TABLE IF EXISTS `log`; CREATE TABLE `log` ( - `id` int(11) unsigned NOT NULL auto_increment, + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `inode` int(11) unsigned NOT NULL, `size` int(11) unsigned NOT NULL, - `name` varchar(30) NOT NULL default '', - `created` datetime default NULL, - PRIMARY KEY (`id`) + `name` varchar(30) NOT NULL DEFAULT '', + `created` datetime DEFAULT NULL, + PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; - # Dump of table message # ------------------------------------------------------------ DROP TABLE IF EXISTS `message`; CREATE TABLE `message` ( - `id` int(11) unsigned NOT NULL auto_increment, + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `file_id` int(10) unsigned NOT NULL, `connect_start` datetime NOT NULL, `ip` int(10) unsigned NOT NULL, `qp_pid` int(10) unsigned NOT NULL, - `result` tinyint(3) NOT NULL default '0', - `distance` mediumint(8) unsigned default NULL, - `time` decimal(3,2) unsigned default NULL, - `os_id` tinyint(3) unsigned default NULL, - `hostname` varchar(128) default NULL, - `helo` varchar(128) default NULL, - `mail_from` varchar(128) default NULL, - `rcpt_to` varchar(128) default NULL, - PRIMARY KEY (`id`), + `result` tinyint(3) NOT NULL DEFAULT '0', + `distance` mediumint(8) unsigned DEFAULT NULL, + `time` decimal(3,2) unsigned DEFAULT NULL, + `os_id` tinyint(3) unsigned DEFAULT NULL, + `hostname` varchar(128) DEFAULT NULL, + `helo` varchar(128) DEFAULT NULL, + `mail_from` varchar(128) DEFAULT NULL, + `rcpt_to` varchar(128) DEFAULT NULL, + PRIMARY KEY (`id`), KEY `file_id` (`file_id`), CONSTRAINT `message_ibfk_1` FOREIGN KEY (`file_id`) REFERENCES `log` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -54,12 +53,12 @@ CREATE TABLE `message` ( DROP TABLE IF EXISTS `message_plugin`; CREATE TABLE `message_plugin` ( - `id` int(11) unsigned NOT NULL auto_increment, + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `msg_id` int(11) unsigned NOT NULL, `plugin_id` int(4) unsigned NOT NULL, `result` tinyint(4) NOT NULL, - `string` varchar(128) default NULL, - PRIMARY KEY (`id`), + `string` varchar(128) DEFAULT NULL, + PRIMARY KEY (`id`), KEY `msg_id` (`msg_id`), KEY `plugin_id` (`plugin_id`), CONSTRAINT `message_plugin_ibfk_1` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, @@ -67,16 +66,15 @@ CREATE TABLE `message_plugin` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; - # Dump of table os # ------------------------------------------------------------ DROP TABLE IF EXISTS `os`; CREATE TABLE `os` ( - `id` tinyint(3) unsigned NOT NULL auto_increment, - `name` varchar(36) default NULL, - PRIMARY KEY (`id`) + `id` tinyint(3) unsigned NOT NULL AUTO_INCREMENT, + `name` varchar(36) DEFAULT NULL, + PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; LOCK TABLES `os` WRITE; @@ -114,81 +112,14 @@ UNLOCK TABLES; DROP TABLE IF EXISTS `plugin`; CREATE TABLE `plugin` ( - `id` int(4) unsigned NOT NULL auto_increment, - `name` varchar(35) character set utf8 NOT NULL default '', - `abb3` char(3) character set utf8 default NULL, - `abb5` char(5) character set utf8 default NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `abb3` (`abb3`), + `id` int(4) unsigned NOT NULL AUTO_INCREMENT, + `name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '', + `abb3` char(3) CHARACTER SET utf8 DEFAULT NULL, + `abb5` char(5) CHARACTER SET utf8 DEFAULT NULL, + PRIMARY KEY (`id`), UNIQUE KEY `abb5` (`abb5`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; -LOCK TABLES `plugin` WRITE; -/*!40000 ALTER TABLE `plugin` DISABLE KEYS */; - -INSERT INTO `plugin` (`id`, `name`, `abb3`, `abb5`) -VALUES - (1,'hosts_allow','alw','allow'), - (2,'ident::geoip','geo','geoip'), - (3,'ident::p0f','p0f',' p0f'), - (5,'karma','krm','karma'), - (6,'dnsbl','dbl','dnsbl'), - (7,'relay','rly','relay'), - (9,'earlytalker','ear','early'), - (15,'helo','hlo','helo'), - (16,'tls','tls',' tls'), - (20,'dont_require_anglebrackets','rab','drabs'), - (21,'unrecognized_commands','cmd','uncmd'), - (22,'noop','nop','noop'), - (23,'random_error','rnd','rande'), - (24,'milter','mtr','mlter'), - (25,'content_log','log','colog'), - (30,'auth::vpopmail_sql','aut','vpsql'), - (31,'auth::vpopmaild','vpd','vpopd'), - (32,'auth::vpopmail','vpo','vpop'), - (33,'auth::checkpasswd','ckp','chkpw'), - (34,'auth::cvs_unix_local','cvs','cvsul'), - (35,'auth::flat_file','flt','aflat'), - (36,'auth::ldap_bind','ldp','aldap'), - (40,'badmailfrom','bmf','badmf'), - (41,'badmailfromto','bmt','bfrto'), - (42,'rhsbl','rbl','rhsbl'), - (44,'resolvable_fromhost','rfh','rsvfh'), - (45,'sender_permitted_from','spf',' spf'), - (50,'badrcptto','bto','badto'), - (51,'rcpt_map','rmp','rcmap'), - (52,'rcpt_regex','rcx','rcrex'), - (53,'qmail_deliverable','qmd',' qmd'), - (55,'rcpt_ok','rok','rcpok'), - (58,'bogus_bounce','bog','bogus'), - (59,'greylisting','gry','greyl'), - (60,'headers','hdr','headr'), - (61,'loop','lop','loop'), - (62,'uribl','uri','uribl'), - (63,'domainkeys','dk','dkey'), - (64,'dkim','dkm','dkim'), - (65,'spamassassin','spm','spama'), - (66,'dspam','dsp','dspam'), - (70,'virus::aveclient','vav','avirs'), - (71,'virus::bitdefender','vbd','bitdf'), - (72,'virus::clamav','cav','clamv'), - (73,'virus::clamdscan','cad','clamd'), - (74,'virus::hbedv','hbv','hbedv'), - (75,'virus::kavscanner','kav','kavsc'), - (76,'virus::klez_filter','klz','vklez'), - (77,'virus::sophie','sop','sophe'), - (78,'virus::uvscan','uvs','uvscn'), - (80,'queue::qmail-queue','qqm','queue'), - (81,'queue::maildir','qdr','qudir'), - (82,'queue::postfix-queue','qpf','qupfx'), - (83,'queue::smtp-forward','qfw','qufwd'), - (84,'queue::exim-bsmtp','qxm','qexim'), - (98,'quit_fortune','for','fortu'), - (99,'connection_time','tim','time'); - -/*!40000 ALTER TABLE `plugin` ENABLE KEYS */; -UNLOCK TABLES; - # Dump of table plugin_aliases # ------------------------------------------------------------ @@ -197,33 +128,10 @@ DROP TABLE IF EXISTS `plugin_aliases`; CREATE TABLE `plugin_aliases` ( `plugin_id` int(11) unsigned NOT NULL, - `name` varchar(35) character set utf8 NOT NULL default '', - KEY `plugin_id` (`plugin_id`), - CONSTRAINT `plugin_id` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON UPDATE CASCADE + `name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '', + UNIQUE KEY `plugin_id` (`plugin_id`,`name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; -LOCK TABLES `plugin_aliases` WRITE; -/*!40000 ALTER TABLE `plugin_aliases` DISABLE KEYS */; - -INSERT INTO `plugin_aliases` (`plugin_id`, `name`) -VALUES - (60,'check_basicheaders'), - (44,'require_resolvable_fromhost'), - (21,'count_unrecognized_commands'), - (9,'check_earlytalker'), - (40,'check_badmailfrom'), - (50,'check_badrcptto'), - (58,'check_bogus_bounce'), - (15,'check_spamhelo'), - (3,'ident::p0f_3a0'), - (80,'queue::qmail_2dqueue'), - (22,'noop_counter'); - -/*!40000 ALTER TABLE `plugin_aliases` ENABLE KEYS */; -UNLOCK TABLES; - - - /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; diff --git a/plugins/registry.txt b/plugins/registry.txt index f02709c..872a239 100644 --- a/plugins/registry.txt +++ b/plugins/registry.txt @@ -1,13 +1,16 @@ # This file contains a list of every plugin used on this server. If you have # additional plugins running, add them here. # Fields are whitespace delimited. Columns are ordered by numeric plugin ID. +# +# the order of plugins in this file determines the order they appear in +# summary output # #id name abb3 abb5 aliases # 201 hosts_allow alw allow 202 ident::geoip geo geoip -203 ident::p0f p0f p0f -204 ident::p0f_3a0 p0f p0f +203 ident::p0f p0f p0f ident::p0f_3a0,ident::p0f_3a1 + 205 karma krm karma 206 dnsbl dbl dnsbl 207 relay rly relay check_relay,check_norelay,relay_only @@ -26,13 +29,13 @@ # # Authentication # -400 auth::auth_vpopmail_sql aut vpsql -401 auth::auth_vpopmaild vpd vpopd -402 auth::auth_vpopmail vpo vpop -403 auth::auth_checkpasswd ckp chkpw -404 auth::auth_cvs_unix_local cvs cvsul -405 auth::auth_flat_file flt aflat -406 auth::auth_ldap_bind ldp aldap +400 auth::auth_vpopmail_sql avq avsql +401 auth::auth_vpopmaild avd vpopd +402 auth::auth_vpopmail avp vpop +403 auth::auth_checkpassword ack chkpw +404 auth::auth_cvs_unix_local acv cvsul +405 auth::auth_flat_file aff aflat +406 auth::auth_ldap_bind ald aldap 407 auth::authdeny dny adeny # # Sender / Envelope From @@ -80,11 +83,11 @@ # # Queue Plugins # -800 queue::qmail-queue qqm queue +800 queue::qmail-queue qqm queue queue::qmail_2dqueue 801 queue::maildir qdr qudir -802 queue::postfix-queue qpf qupfx -803 queue::smtp-forward qfw qufwd -804 queue::exim-bsmtp qxm qexim +802 queue::postfix-queue qpf qupfx queue::postfix_2dqueue +803 queue::smtp-forward qfw qufwd queue::smtp_2dqueue +804 queue::exim-bsmtp qxm qexim queue::exim_2dbsmtp 900 quit_fortune for fortu