log2sql: populate plugins table from registry.txt
much easier for local customizations. moved SQL connection settings to config/log2sql
This commit is contained in:
Matt Simerson
parent
c6b5a0dfae
commit
ca678ba736
4
config.sample/log2sql
Normal file
4
config.sample/log2sql
Normal file
@ -0,0 +1,4 @@
|
||||
# comments are allowed
|
||||
dsn = DBI:mysql:database=qpsmtpd;host=db;port=3306
|
||||
user = qplog
|
||||
pass = can mysql have 6 spaces in a passphrase?
|
||||
74
log/log2sql
74
log/log2sql
@ -6,21 +6,19 @@ use warnings;
|
||||
use Cwd;
|
||||
use Data::Dumper;
|
||||
use DBIx::Simple;
|
||||
use IO::File;
|
||||
use File::stat;
|
||||
use Time::TAI64 qw/ tai2unix /;
|
||||
|
||||
$Data::Dumper::Sortkeys = 1;
|
||||
|
||||
my $dsn = 'DBI:mysql:database=qpsmtpd;host=db;port=3306';
|
||||
my $user = 'qplog';
|
||||
my $pass = 't0ps3cret';
|
||||
|
||||
my $logdir = get_log_dir();
|
||||
my @logfiles = get_logfiles($logdir);
|
||||
|
||||
my (%plugins, %os, %message_ids);
|
||||
my $has_cleanup;
|
||||
my $db = get_db();
|
||||
check_plugins_table();
|
||||
|
||||
foreach my $file (@logfiles) {
|
||||
my ($fid, $offset) = check_logfile($file);
|
||||
@ -208,6 +206,7 @@ sub parse_logfile {
|
||||
#warn "type: $type\n";
|
||||
if ($type eq 'plugin') {
|
||||
next if $plugin eq 'naughty'; # housekeeping only
|
||||
next if $plugin eq 'karma' && 'karma adjust' eq substr($message,0,12);
|
||||
insert_plugin($msg_id, $plugin, $message);
|
||||
}
|
||||
elsif ($type eq 'queue') {
|
||||
@ -529,12 +528,70 @@ sub get_score {
|
||||
|
||||
sub get_db {
|
||||
|
||||
my $db = DBIx::Simple->connect($dsn, $user, $pass)
|
||||
my %dbv = get_config('log2sql');
|
||||
|
||||
$dbv{dsn} ||= 'DBI:mysql:database=qpsmtpd;host=db;port=3306';
|
||||
$dbv{user} ||= 'qplog';
|
||||
$dbv{pass} ||= 't0ps3cret';
|
||||
|
||||
print Dumper(\%dbv);
|
||||
my $db = DBIx::Simple->connect($dbv{dsn}, $dbv{user}, $dbv{pass})
|
||||
or die DBIx::Simple->error;
|
||||
|
||||
return $db;
|
||||
}
|
||||
|
||||
sub get_config {
|
||||
my $file = shift or die "missing file name\n";
|
||||
my %values;
|
||||
foreach my $line ( get_config_contents( $file ) ) {
|
||||
next if $line =~ /^#/;
|
||||
chomp $line;
|
||||
my ($key,$val) = split /\s*=\s*/, $line, 2;
|
||||
$values{$key} = $val;
|
||||
};
|
||||
return %values;
|
||||
};
|
||||
|
||||
sub get_config_contents {
|
||||
my $name = shift;
|
||||
|
||||
my @config_dirs = qw[ config ../config log plugins ];
|
||||
foreach my $dir ( @config_dirs ) {
|
||||
next if ! -f "$dir/$name";
|
||||
|
||||
my $fh = IO::File->new();
|
||||
if ( ! $fh->open( "$dir/$name", '<' ) ) {
|
||||
warn "unable to open config file $dir/$name\n";
|
||||
next;
|
||||
};
|
||||
my @contents = <$fh>;
|
||||
return @contents;
|
||||
};
|
||||
};
|
||||
|
||||
sub check_plugins_table {
|
||||
my $rows = exec_query( 'SELECT COUNT(*) FROM plugin');
|
||||
return if scalar @$rows != 0;
|
||||
my @lines = get_config_contents('registry.txt');
|
||||
foreach my $line ( @lines ) {
|
||||
next if $line =~ /^\s*#/; # ignore comments
|
||||
chomp $line;
|
||||
next if ! $line;
|
||||
my ($id, $name, $abb3, $abb5, $aliases) = split /\s+/, $line, 5;
|
||||
my $q = "REPLACE INTO plugin (id,name,abb3,abb5) VALUES (??)";
|
||||
print "query: $q, $id, $name, $abb3, $abb5\n";
|
||||
exec_query($q, [$id, $name, $abb3, $abb5 ]);
|
||||
next if ! $aliases;
|
||||
foreach my $alias ( split /\s*,\s*/, $aliases ) {
|
||||
next if ! $alias;
|
||||
my $aq = "REPLACE INTO plugin_aliases (plugin_id,name) VALUES (??)";
|
||||
print "aqury: $aq, $id, $alias\n";
|
||||
exec_query($aq, [$id, $alias]);
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
sub exec_query {
|
||||
my $query = shift;
|
||||
my $params = shift;
|
||||
@ -550,10 +607,11 @@ sub exec_query {
|
||||
}
|
||||
|
||||
#warn "err: $err\n";
|
||||
if ($query =~ /INSERT INTO/) {
|
||||
my ($table) = $query =~ /INSERT INTO (\w+)\s/;
|
||||
if ($query =~ /(?:REPLACE|INSERT) INTO/) {
|
||||
my ($table) = $query =~ /(?:REPLACE|INSERT) INTO (\w+)\s/;
|
||||
$db->query($query, @params);
|
||||
die "$db->error\n$err" if $db->error ne 'DBI error: ';
|
||||
warn "$db->error\n$err" if $db->error ne 'DBI error: ';
|
||||
return if $query =~ /^REPLACE/;
|
||||
my $id = $db->last_insert_id(undef, undef, $table, undef) or die $err;
|
||||
return $id;
|
||||
}
|
||||
|
||||
146
log/log2sql.sql
146
log/log2sql.sql
@ -13,35 +13,34 @@
|
||||
DROP TABLE IF EXISTS `log`;
|
||||
|
||||
CREATE TABLE `log` (
|
||||
`id` int(11) unsigned NOT NULL auto_increment,
|
||||
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`inode` int(11) unsigned NOT NULL,
|
||||
`size` int(11) unsigned NOT NULL,
|
||||
`name` varchar(30) NOT NULL default '',
|
||||
`created` datetime default NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
`name` varchar(30) NOT NULL DEFAULT '',
|
||||
`created` datetime DEFAULT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
|
||||
# Dump of table message
|
||||
# ------------------------------------------------------------
|
||||
|
||||
DROP TABLE IF EXISTS `message`;
|
||||
|
||||
CREATE TABLE `message` (
|
||||
`id` int(11) unsigned NOT NULL auto_increment,
|
||||
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`file_id` int(10) unsigned NOT NULL,
|
||||
`connect_start` datetime NOT NULL,
|
||||
`ip` int(10) unsigned NOT NULL,
|
||||
`qp_pid` int(10) unsigned NOT NULL,
|
||||
`result` tinyint(3) NOT NULL default '0',
|
||||
`distance` mediumint(8) unsigned default NULL,
|
||||
`time` decimal(3,2) unsigned default NULL,
|
||||
`os_id` tinyint(3) unsigned default NULL,
|
||||
`hostname` varchar(128) default NULL,
|
||||
`helo` varchar(128) default NULL,
|
||||
`mail_from` varchar(128) default NULL,
|
||||
`rcpt_to` varchar(128) default NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
`result` tinyint(3) NOT NULL DEFAULT '0',
|
||||
`distance` mediumint(8) unsigned DEFAULT NULL,
|
||||
`time` decimal(3,2) unsigned DEFAULT NULL,
|
||||
`os_id` tinyint(3) unsigned DEFAULT NULL,
|
||||
`hostname` varchar(128) DEFAULT NULL,
|
||||
`helo` varchar(128) DEFAULT NULL,
|
||||
`mail_from` varchar(128) DEFAULT NULL,
|
||||
`rcpt_to` varchar(128) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `file_id` (`file_id`),
|
||||
CONSTRAINT `message_ibfk_1` FOREIGN KEY (`file_id`) REFERENCES `log` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
@ -54,12 +53,12 @@ CREATE TABLE `message` (
|
||||
DROP TABLE IF EXISTS `message_plugin`;
|
||||
|
||||
CREATE TABLE `message_plugin` (
|
||||
`id` int(11) unsigned NOT NULL auto_increment,
|
||||
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`msg_id` int(11) unsigned NOT NULL,
|
||||
`plugin_id` int(4) unsigned NOT NULL,
|
||||
`result` tinyint(4) NOT NULL,
|
||||
`string` varchar(128) default NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
`string` varchar(128) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `msg_id` (`msg_id`),
|
||||
KEY `plugin_id` (`plugin_id`),
|
||||
CONSTRAINT `message_plugin_ibfk_1` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
@ -67,16 +66,15 @@ CREATE TABLE `message_plugin` (
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
|
||||
|
||||
# Dump of table os
|
||||
# ------------------------------------------------------------
|
||||
|
||||
DROP TABLE IF EXISTS `os`;
|
||||
|
||||
CREATE TABLE `os` (
|
||||
`id` tinyint(3) unsigned NOT NULL auto_increment,
|
||||
`name` varchar(36) default NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
`id` tinyint(3) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(36) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
|
||||
LOCK TABLES `os` WRITE;
|
||||
@ -114,81 +112,14 @@ UNLOCK TABLES;
|
||||
DROP TABLE IF EXISTS `plugin`;
|
||||
|
||||
CREATE TABLE `plugin` (
|
||||
`id` int(4) unsigned NOT NULL auto_increment,
|
||||
`name` varchar(35) character set utf8 NOT NULL default '',
|
||||
`abb3` char(3) character set utf8 default NULL,
|
||||
`abb5` char(5) character set utf8 default NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `abb3` (`abb3`),
|
||||
`id` int(4) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '',
|
||||
`abb3` char(3) CHARACTER SET utf8 DEFAULT NULL,
|
||||
`abb5` char(5) CHARACTER SET utf8 DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `abb5` (`abb5`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
|
||||
|
||||
LOCK TABLES `plugin` WRITE;
|
||||
/*!40000 ALTER TABLE `plugin` DISABLE KEYS */;
|
||||
|
||||
INSERT INTO `plugin` (`id`, `name`, `abb3`, `abb5`)
|
||||
VALUES
|
||||
(1,'hosts_allow','alw','allow'),
|
||||
(2,'ident::geoip','geo','geoip'),
|
||||
(3,'ident::p0f','p0f',' p0f'),
|
||||
(5,'karma','krm','karma'),
|
||||
(6,'dnsbl','dbl','dnsbl'),
|
||||
(7,'relay','rly','relay'),
|
||||
(9,'earlytalker','ear','early'),
|
||||
(15,'helo','hlo','helo'),
|
||||
(16,'tls','tls',' tls'),
|
||||
(20,'dont_require_anglebrackets','rab','drabs'),
|
||||
(21,'unrecognized_commands','cmd','uncmd'),
|
||||
(22,'noop','nop','noop'),
|
||||
(23,'random_error','rnd','rande'),
|
||||
(24,'milter','mtr','mlter'),
|
||||
(25,'content_log','log','colog'),
|
||||
(30,'auth::vpopmail_sql','aut','vpsql'),
|
||||
(31,'auth::vpopmaild','vpd','vpopd'),
|
||||
(32,'auth::vpopmail','vpo','vpop'),
|
||||
(33,'auth::checkpasswd','ckp','chkpw'),
|
||||
(34,'auth::cvs_unix_local','cvs','cvsul'),
|
||||
(35,'auth::flat_file','flt','aflat'),
|
||||
(36,'auth::ldap_bind','ldp','aldap'),
|
||||
(40,'badmailfrom','bmf','badmf'),
|
||||
(41,'badmailfromto','bmt','bfrto'),
|
||||
(42,'rhsbl','rbl','rhsbl'),
|
||||
(44,'resolvable_fromhost','rfh','rsvfh'),
|
||||
(45,'sender_permitted_from','spf',' spf'),
|
||||
(50,'badrcptto','bto','badto'),
|
||||
(51,'rcpt_map','rmp','rcmap'),
|
||||
(52,'rcpt_regex','rcx','rcrex'),
|
||||
(53,'qmail_deliverable','qmd',' qmd'),
|
||||
(55,'rcpt_ok','rok','rcpok'),
|
||||
(58,'bogus_bounce','bog','bogus'),
|
||||
(59,'greylisting','gry','greyl'),
|
||||
(60,'headers','hdr','headr'),
|
||||
(61,'loop','lop','loop'),
|
||||
(62,'uribl','uri','uribl'),
|
||||
(63,'domainkeys','dk','dkey'),
|
||||
(64,'dkim','dkm','dkim'),
|
||||
(65,'spamassassin','spm','spama'),
|
||||
(66,'dspam','dsp','dspam'),
|
||||
(70,'virus::aveclient','vav','avirs'),
|
||||
(71,'virus::bitdefender','vbd','bitdf'),
|
||||
(72,'virus::clamav','cav','clamv'),
|
||||
(73,'virus::clamdscan','cad','clamd'),
|
||||
(74,'virus::hbedv','hbv','hbedv'),
|
||||
(75,'virus::kavscanner','kav','kavsc'),
|
||||
(76,'virus::klez_filter','klz','vklez'),
|
||||
(77,'virus::sophie','sop','sophe'),
|
||||
(78,'virus::uvscan','uvs','uvscn'),
|
||||
(80,'queue::qmail-queue','qqm','queue'),
|
||||
(81,'queue::maildir','qdr','qudir'),
|
||||
(82,'queue::postfix-queue','qpf','qupfx'),
|
||||
(83,'queue::smtp-forward','qfw','qufwd'),
|
||||
(84,'queue::exim-bsmtp','qxm','qexim'),
|
||||
(98,'quit_fortune','for','fortu'),
|
||||
(99,'connection_time','tim','time');
|
||||
|
||||
/*!40000 ALTER TABLE `plugin` ENABLE KEYS */;
|
||||
UNLOCK TABLES;
|
||||
|
||||
|
||||
# Dump of table plugin_aliases
|
||||
# ------------------------------------------------------------
|
||||
@ -197,33 +128,10 @@ DROP TABLE IF EXISTS `plugin_aliases`;
|
||||
|
||||
CREATE TABLE `plugin_aliases` (
|
||||
`plugin_id` int(11) unsigned NOT NULL,
|
||||
`name` varchar(35) character set utf8 NOT NULL default '',
|
||||
KEY `plugin_id` (`plugin_id`),
|
||||
CONSTRAINT `plugin_id` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON UPDATE CASCADE
|
||||
`name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '',
|
||||
UNIQUE KEY `plugin_id` (`plugin_id`,`name`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
|
||||
|
||||
LOCK TABLES `plugin_aliases` WRITE;
|
||||
/*!40000 ALTER TABLE `plugin_aliases` DISABLE KEYS */;
|
||||
|
||||
INSERT INTO `plugin_aliases` (`plugin_id`, `name`)
|
||||
VALUES
|
||||
(60,'check_basicheaders'),
|
||||
(44,'require_resolvable_fromhost'),
|
||||
(21,'count_unrecognized_commands'),
|
||||
(9,'check_earlytalker'),
|
||||
(40,'check_badmailfrom'),
|
||||
(50,'check_badrcptto'),
|
||||
(58,'check_bogus_bounce'),
|
||||
(15,'check_spamhelo'),
|
||||
(3,'ident::p0f_3a0'),
|
||||
(80,'queue::qmail_2dqueue'),
|
||||
(22,'noop_counter');
|
||||
|
||||
/*!40000 ALTER TABLE `plugin_aliases` ENABLE KEYS */;
|
||||
UNLOCK TABLES;
|
||||
|
||||
|
||||
|
||||
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
|
||||
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
|
||||
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
|
||||
|
||||
@ -1,13 +1,16 @@
|
||||
# This file contains a list of every plugin used on this server. If you have
|
||||
# additional plugins running, add them here.
|
||||
# Fields are whitespace delimited. Columns are ordered by numeric plugin ID.
|
||||
#
|
||||
# the order of plugins in this file determines the order they appear in
|
||||
# summary output
|
||||
#
|
||||
#id name abb3 abb5 aliases
|
||||
#
|
||||
201 hosts_allow alw allow
|
||||
202 ident::geoip geo geoip
|
||||
203 ident::p0f p0f p0f
|
||||
204 ident::p0f_3a0 p0f p0f
|
||||
203 ident::p0f p0f p0f ident::p0f_3a0,ident::p0f_3a1
|
||||
|
||||
205 karma krm karma
|
||||
206 dnsbl dbl dnsbl
|
||||
207 relay rly relay check_relay,check_norelay,relay_only
|
||||
@ -26,13 +29,13 @@
|
||||
#
|
||||
# Authentication
|
||||
#
|
||||
400 auth::auth_vpopmail_sql aut vpsql
|
||||
401 auth::auth_vpopmaild vpd vpopd
|
||||
402 auth::auth_vpopmail vpo vpop
|
||||
403 auth::auth_checkpasswd ckp chkpw
|
||||
404 auth::auth_cvs_unix_local cvs cvsul
|
||||
405 auth::auth_flat_file flt aflat
|
||||
406 auth::auth_ldap_bind ldp aldap
|
||||
400 auth::auth_vpopmail_sql avq avsql
|
||||
401 auth::auth_vpopmaild avd vpopd
|
||||
402 auth::auth_vpopmail avp vpop
|
||||
403 auth::auth_checkpassword ack chkpw
|
||||
404 auth::auth_cvs_unix_local acv cvsul
|
||||
405 auth::auth_flat_file aff aflat
|
||||
406 auth::auth_ldap_bind ald aldap
|
||||
407 auth::authdeny dny adeny
|
||||
#
|
||||
# Sender / Envelope From
|
||||
@ -80,11 +83,11 @@
|
||||
#
|
||||
# Queue Plugins
|
||||
#
|
||||
800 queue::qmail-queue qqm queue
|
||||
800 queue::qmail-queue qqm queue queue::qmail_2dqueue
|
||||
801 queue::maildir qdr qudir
|
||||
802 queue::postfix-queue qpf qupfx
|
||||
803 queue::smtp-forward qfw qufwd
|
||||
804 queue::exim-bsmtp qxm qexim
|
||||
802 queue::postfix-queue qpf qupfx queue::postfix_2dqueue
|
||||
803 queue::smtp-forward qfw qufwd queue::smtp_2dqueue
|
||||
804 queue::exim-bsmtp qxm qexim queue::exim_2dbsmtp
|
||||
|
||||
900 quit_fortune for fortu
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user