From b7ce45a5020bcc0e2ffe6c73e63036a04c73f5d9 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Sun, 21 Apr 2013 16:06:37 -0400
Subject: [PATCH 01/21] moved tls plugin to the top of the config

it must be listed before other connection plugins for port 465
place it up there just in case
---
 config.sample/plugins | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/config.sample/plugins b/config.sample/plugins
index e59bcae..bb15895 100644
--- a/config.sample/plugins
+++ b/config.sample/plugins
@@ -6,6 +6,10 @@
 # plugins/http_config for details.
 #   http_config http://localhost/~smtpd/config/  http://www.example.com/smtp.pl?config=
 
+# tls should load before count_unrecognized_commands
+# to support legacy port 465, tls must load before connection plugins
+#tls
+
 # hosts_allow does not work with the tcpserver deployment model!
 #   perldoc plugins/hosts_allow for an alternative.
 #
@@ -23,8 +27,6 @@ ident/geoip
 fcrdns
 
 quit_fortune
-# tls should load before count_unrecognized_commands
-#tls
 earlytalker
 count_unrecognized_commands 4
 

From 7d88c51e0a980a3daf0d3e85cf9a47cdc49ee711 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Sun, 21 Apr 2013 17:02:34 -0400
Subject: [PATCH 02/21] auth_chkpw: added pass|fail prefix to log msgs

---
 plugins/auth/auth_checkpassword | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/auth/auth_checkpassword b/plugins/auth/auth_checkpassword
index cb84758..a20fb71 100644
--- a/plugins/auth/auth_checkpassword
+++ b/plugins/auth/auth_checkpassword
@@ -136,11 +136,12 @@ sub auth_checkpassword {
     my $status = $?;
 
     if ($status != 0) {
-        $self->log(LOGNOTICE, "authentication failed ($status)");
+        $self->log(LOGNOTICE, "fail, auth failed: $status");
         return (DECLINED);
     }
 
     $self->connection->notes('authuser', $user);
+    $self->log(LOGINFO, "pass, auth success with $method");
     return (OK, "auth_checkpassword");
 }
 

From 71997439c147c2d7dcc363c9a278a1897d4d9765 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Sun, 21 Apr 2013 17:03:24 -0400
Subject: [PATCH 03/21] tls: added pass|fail prefix to a couple log msgs

---
 plugins/tls | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/plugins/tls b/plugins/tls
index 533c5df..4aceaad 100644
--- a/plugins/tls
+++ b/plugins/tls
@@ -149,12 +149,16 @@ sub hook_connect {
     my ($self, $transaction) = @_;
 
     my $local_port = $self->qp->connection->local_port;
-    return DECLINED unless defined $local_port && $local_port == 465;    # SMTPS
+    if ( ! defined $local_port || $local_port != 465 ) {  # SMTPS
+        $self->log(LOGDEBUG, "skip, not SMTPS");
+        return DECLINED;
+    };
 
     unless (_convert_to_ssl($self)) {
+        $self->log(LOGINFO, "fail, unable to establish SSL");
         return (DENY_DISCONNECT, "Cannot establish SSL session");
     }
-    $self->log(LOGWARN, "Connected via SMTPS");
+    $self->log(LOGINFO, "pass, connect via SMTPS");
     return DECLINED;
 }
 

From f1aa848166d94313ea84c761f9cae14d72834241 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Sun, 21 Apr 2013 19:54:06 -0400
Subject: [PATCH 04/21] dkim: reduce INFO logging to once per connect

---
 plugins/dkim | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/plugins/dkim b/plugins/dkim
index dbef7a7..13815a1 100644
--- a/plugins/dkim
+++ b/plugins/dkim
@@ -353,7 +353,7 @@ sub handle_sig_pass {
     elsif ($prs->{neutral}) {
         $self->add_header($mess);
         $self->log(LOGINFO, "pass, valid signature, neutral policy");
-        $self->log(LOGINFO, $mess);
+        $self->log(LOGDEBUG, $mess);
         return DECLINED;
     }
     elsif ($prs->{reject}) {
@@ -364,7 +364,7 @@ sub handle_sig_pass {
                             "fail, valid sig, reject policy");
     }
 
-    # this should never happen
+    # this should never happen,
     $self->add_header($mess);
     $self->log(LOGERROR, "pass, valid sig, no policy results");
     $self->log(LOGINFO,  $mess);
@@ -449,14 +449,17 @@ sub get_keydir {
 sub save_signatures_to_note {
     my ($self, $dkim) = @_;
 
+    my %domains;
     foreach my $sig ($dkim->signatures) {
         next if $sig->result ne 'pass';
-        my $doms = $self->connection->notes('dkim_pass_domains') || [];
-        next if grep /$sig->domain/, @$doms;  # already in the list
-        push @$doms, $sig->domain;
-        $self->connection->notes('dkim_pass_domains', $doms);
-        $self->log(LOGINFO, "info, added " . $sig->domain);
+        $domains{$sig->domain} = 1;
     }
+    return if 0 == scalar keys %domains;
+
+    my $doms = $self->connection->notes('dkim_pass_domains') || [];
+    push @$doms, keys %domains;
+    $self->log(LOGDEBUG, "info, signed by: ". join(',', keys %domains) );
+    $self->connection->notes('dkim_pass_domains', $doms);
 }
 
 sub send_message_to_dkim {

From 8b95e9053d1e69b6c29e62696c483302e04e5c56 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Sun, 21 Apr 2013 20:33:46 -0400
Subject: [PATCH 05/21] Makefile.PL: gzip -9, and clean up test db

and a perltidy
---
 Makefile.PL | 40 ++++++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/Makefile.PL b/Makefile.PL
index 3a40c1b..39d9104 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -4,21 +4,25 @@ use strict;
 use ExtUtils::MakeMaker;
 
 WriteMakefile(
-    NAME           => 'qpsmtpd',
-    VERSION_FROM   => 'lib/Qpsmtpd.pm',
-    PREREQ_PM      => {
-        'Mail::Header' => 0,
-        'MIME::Base64' => 0,
-        'Net::DNS'     => 0.39,
-        'Data::Dumper' => 0,
-        'File::Temp'   => 0,
-        'Time::HiRes'  => 0,
-        'Net::IP'      => 0,
-        'Date::Parse'  => 0,
-    },
-    ABSTRACT       => 'Flexible smtpd daemon written in Perl',
-    AUTHOR         => 'Ask Bjoern Hansen <ask@develooper.com>',
-    EXE_FILES      => [qw(qpsmtpd qpsmtpd-forkserver qpsmtpd-prefork qpsmtpd-async)],
+    NAME         => 'qpsmtpd',
+    VERSION_FROM => 'lib/Qpsmtpd.pm',
+    PREREQ_PM    => {
+                  'Mail::Header' => 0,
+                  'MIME::Base64' => 0,
+                  'Net::DNS'     => 0.39,
+                  'Data::Dumper' => 0,
+                  'File::Temp'   => 0,
+                  'Time::HiRes'  => 0,
+                  'Net::IP'      => 0,
+                  'Date::Parse'  => 0,
+                 },
+    ABSTRACT  => 'Flexible smtpd daemon written in Perl',
+    AUTHOR    => 'Ask Bjoern Hansen <ask@develooper.com>',
+    EXE_FILES => [qw(qpsmtpd qpsmtpd-forkserver qpsmtpd-prefork qpsmtpd-async)],
+    dist      => {COMPRESS => 'gzip -9f',},
+    clean     => {
+              FILES => ['t/config/greylist.dbm*',],
+             },
 );
 
 sub MY::libscan {
@@ -28,11 +32,11 @@ sub MY::libscan {
 }
 
 sub MY::postamble {
-  qq[
+    qq[
 testcover :
 \t cover -delete && \\
-   HARNESS_PERL_SWITCHES=-MDevel::Cover \$(MAKE) test && \\
-   cover
+	HARNESS_PERL_SWITCHES=-MDevel::Cover \$(MAKE) test && \\
+	cover
 ]
 
 }

From 78ac01df760f23e88f70fb44b67c8642b7d2d7e6 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Mon, 22 Apr 2013 02:12:53 -0400
Subject: [PATCH 06/21] log2sql: populate plugins table from registry.txt

much easier for local customizations.
moved SQL connection settings to config/log2sql
---
 config.sample/log2sql |   4 ++
 log/log2sql           |  74 ++++++++++++++++++---
 log/log2sql.sql       | 146 ++++++++----------------------------------
 plugins/registry.txt  |  29 +++++----
 4 files changed, 113 insertions(+), 140 deletions(-)
 create mode 100644 config.sample/log2sql

diff --git a/config.sample/log2sql b/config.sample/log2sql
new file mode 100644
index 0000000..5b02654
--- /dev/null
+++ b/config.sample/log2sql
@@ -0,0 +1,4 @@
+# comments are allowed
+dsn  = DBI:mysql:database=qpsmtpd;host=db;port=3306
+user = qplog
+pass = can mysql have 6 spaces in a passphrase?
diff --git a/log/log2sql b/log/log2sql
index fa8010e..89bb1f1 100755
--- a/log/log2sql
+++ b/log/log2sql
@@ -6,21 +6,19 @@ use warnings;
 use Cwd;
 use Data::Dumper;
 use DBIx::Simple;
+use IO::File;
 use File::stat;
 use Time::TAI64 qw/ tai2unix /;
 
 $Data::Dumper::Sortkeys = 1;
 
-my $dsn  = 'DBI:mysql:database=qpsmtpd;host=db;port=3306';
-my $user = 'qplog';
-my $pass = 't0ps3cret';
-
 my $logdir   = get_log_dir();
 my @logfiles = get_logfiles($logdir);
 
 my (%plugins, %os, %message_ids);
 my $has_cleanup;
 my $db = get_db();
+check_plugins_table();
 
 foreach my $file (@logfiles) {
     my ($fid, $offset) = check_logfile($file);
@@ -208,6 +206,7 @@ sub parse_logfile {
         #warn "type: $type\n";
         if ($type eq 'plugin') {
             next if $plugin eq 'naughty';             # housekeeping only
+            next if $plugin eq 'karma' && 'karma adjust' eq substr($message,0,12);
             insert_plugin($msg_id, $plugin, $message);
         }
         elsif ($type eq 'queue') {
@@ -529,12 +528,70 @@ sub get_score {
 
 sub get_db {
 
-    my $db = DBIx::Simple->connect($dsn, $user, $pass)
+    my %dbv = get_config('log2sql');
+
+    $dbv{dsn}  ||= 'DBI:mysql:database=qpsmtpd;host=db;port=3306';
+    $dbv{user} ||= 'qplog';
+    $dbv{pass} ||= 't0ps3cret';
+
+    print Dumper(\%dbv);
+    my $db = DBIx::Simple->connect($dbv{dsn}, $dbv{user}, $dbv{pass})
       or die DBIx::Simple->error;
 
     return $db;
 }
 
+sub get_config {
+    my $file = shift or die "missing file name\n";
+    my %values;
+    foreach my $line ( get_config_contents( $file ) ) {
+        next if $line =~ /^#/;
+        chomp $line;
+        my ($key,$val) = split /\s*=\s*/, $line, 2;
+        $values{$key} = $val;
+    };
+    return %values;
+};
+
+sub get_config_contents {
+    my $name = shift;
+
+    my @config_dirs = qw[ config ../config log plugins ];
+    foreach my $dir ( @config_dirs ) {
+        next if ! -f "$dir/$name";
+
+        my $fh = IO::File->new();
+        if ( ! $fh->open( "$dir/$name", '<' ) ) {
+            warn "unable to open config file $dir/$name\n";
+            next;
+        };
+        my @contents = <$fh>;
+        return @contents;
+    };
+};
+
+sub check_plugins_table {
+    my $rows = exec_query( 'SELECT COUNT(*) FROM plugin');
+    return if scalar @$rows != 0;
+    my @lines = get_config_contents('registry.txt');
+    foreach my $line ( @lines ) {
+        next if $line =~ /^\s*#/;  # ignore comments
+        chomp $line;
+        next if ! $line;
+        my ($id, $name, $abb3, $abb5, $aliases) = split /\s+/, $line, 5;
+        my $q = "REPLACE INTO plugin (id,name,abb3,abb5) VALUES (??)";
+        print "query: $q, $id, $name, $abb3, $abb5\n";
+        exec_query($q, [$id, $name, $abb3, $abb5 ]);
+        next if ! $aliases;
+        foreach my $alias ( split /\s*,\s*/, $aliases ) {
+            next if ! $alias;
+            my $aq = "REPLACE INTO plugin_aliases (plugin_id,name) VALUES (??)";
+            print "aqury: $aq, $id, $alias\n";
+            exec_query($aq, [$id, $alias]);
+        };
+    };
+};
+
 sub exec_query {
     my $query  = shift;
     my $params = shift;
@@ -550,10 +607,11 @@ sub exec_query {
     }
 
     #warn "err: $err\n";
-    if ($query =~ /INSERT INTO/) {
-        my ($table) = $query =~ /INSERT INTO (\w+)\s/;
+    if ($query =~ /(?:REPLACE|INSERT) INTO/) {
+        my ($table) = $query =~ /(?:REPLACE|INSERT) INTO (\w+)\s/;
         $db->query($query, @params);
-        die "$db->error\n$err" if $db->error ne 'DBI error: ';
+        warn "$db->error\n$err" if $db->error ne 'DBI error: ';
+        return if $query =~ /^REPLACE/;
         my $id = $db->last_insert_id(undef, undef, $table, undef) or die $err;
         return $id;
     }
diff --git a/log/log2sql.sql b/log/log2sql.sql
index 4f975eb..0c06f35 100644
--- a/log/log2sql.sql
+++ b/log/log2sql.sql
@@ -13,35 +13,34 @@
 DROP TABLE IF EXISTS `log`;
 
 CREATE TABLE `log` (
-  `id` int(11) unsigned NOT NULL auto_increment,
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `inode` int(11) unsigned NOT NULL,
   `size` int(11) unsigned NOT NULL,
-  `name` varchar(30) NOT NULL default '',
-  `created` datetime default NULL,
-  PRIMARY KEY  (`id`)
+  `name` varchar(30) NOT NULL DEFAULT '',
+  `created` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-
 # Dump of table message
 # ------------------------------------------------------------
 
 DROP TABLE IF EXISTS `message`;
 
 CREATE TABLE `message` (
-  `id` int(11) unsigned NOT NULL auto_increment,
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `file_id` int(10) unsigned NOT NULL,
   `connect_start` datetime NOT NULL,
   `ip` int(10) unsigned NOT NULL,
   `qp_pid` int(10) unsigned NOT NULL,
-  `result` tinyint(3) NOT NULL default '0',
-  `distance` mediumint(8) unsigned default NULL,
-  `time` decimal(3,2) unsigned default NULL,
-  `os_id` tinyint(3) unsigned default NULL,
-  `hostname` varchar(128) default NULL,
-  `helo` varchar(128) default NULL,
-  `mail_from` varchar(128) default NULL,
-  `rcpt_to` varchar(128) default NULL,
-  PRIMARY KEY  (`id`),
+  `result` tinyint(3) NOT NULL DEFAULT '0',
+  `distance` mediumint(8) unsigned DEFAULT NULL,
+  `time` decimal(3,2) unsigned DEFAULT NULL,
+  `os_id` tinyint(3) unsigned DEFAULT NULL,
+  `hostname` varchar(128) DEFAULT NULL,
+  `helo` varchar(128) DEFAULT NULL,
+  `mail_from` varchar(128) DEFAULT NULL,
+  `rcpt_to` varchar(128) DEFAULT NULL,
+  PRIMARY KEY (`id`),
   KEY `file_id` (`file_id`),
   CONSTRAINT `message_ibfk_1` FOREIGN KEY (`file_id`) REFERENCES `log` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@@ -54,12 +53,12 @@ CREATE TABLE `message` (
 DROP TABLE IF EXISTS `message_plugin`;
 
 CREATE TABLE `message_plugin` (
-  `id` int(11) unsigned NOT NULL auto_increment,
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `msg_id` int(11) unsigned NOT NULL,
   `plugin_id` int(4) unsigned NOT NULL,
   `result` tinyint(4) NOT NULL,
-  `string` varchar(128) default NULL,
-  PRIMARY KEY  (`id`),
+  `string` varchar(128) DEFAULT NULL,
+  PRIMARY KEY (`id`),
   KEY `msg_id` (`msg_id`),
   KEY `plugin_id` (`plugin_id`),
   CONSTRAINT `message_plugin_ibfk_1` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
@@ -67,16 +66,15 @@ CREATE TABLE `message_plugin` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 
-
 # Dump of table os
 # ------------------------------------------------------------
 
 DROP TABLE IF EXISTS `os`;
 
 CREATE TABLE `os` (
-  `id` tinyint(3) unsigned NOT NULL auto_increment,
-  `name` varchar(36) default NULL,
-  PRIMARY KEY  (`id`)
+  `id` tinyint(3) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(36) DEFAULT NULL,
+  PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 LOCK TABLES `os` WRITE;
@@ -114,81 +112,14 @@ UNLOCK TABLES;
 DROP TABLE IF EXISTS `plugin`;
 
 CREATE TABLE `plugin` (
-  `id` int(4) unsigned NOT NULL auto_increment,
-  `name` varchar(35) character set utf8 NOT NULL default '',
-  `abb3` char(3) character set utf8 default NULL,
-  `abb5` char(5) character set utf8 default NULL,
-  PRIMARY KEY  (`id`),
-  UNIQUE KEY `abb3` (`abb3`),
+  `id` int(4) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '',
+  `abb3` char(3) CHARACTER SET utf8 DEFAULT NULL,
+  `abb5` char(5) CHARACTER SET utf8 DEFAULT NULL,
+  PRIMARY KEY (`id`),
   UNIQUE KEY `abb5` (`abb5`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
 
-LOCK TABLES `plugin` WRITE;
-/*!40000 ALTER TABLE `plugin` DISABLE KEYS */;
-
-INSERT INTO `plugin` (`id`, `name`, `abb3`, `abb5`)
-VALUES
-    (1,'hosts_allow','alw','allow'),
-    (2,'ident::geoip','geo','geoip'),
-    (3,'ident::p0f','p0f',' p0f'),
-    (5,'karma','krm','karma'),
-    (6,'dnsbl','dbl','dnsbl'),
-    (7,'relay','rly','relay'),
-    (9,'earlytalker','ear','early'),
-    (15,'helo','hlo','helo'),
-    (16,'tls','tls',' tls'),
-    (20,'dont_require_anglebrackets','rab','drabs'),
-    (21,'unrecognized_commands','cmd','uncmd'),
-    (22,'noop','nop','noop'),
-    (23,'random_error','rnd','rande'),
-    (24,'milter','mtr','mlter'),
-    (25,'content_log','log','colog'),
-    (30,'auth::vpopmail_sql','aut','vpsql'),
-    (31,'auth::vpopmaild','vpd','vpopd'),
-    (32,'auth::vpopmail','vpo','vpop'),
-    (33,'auth::checkpasswd','ckp','chkpw'),
-    (34,'auth::cvs_unix_local','cvs','cvsul'),
-    (35,'auth::flat_file','flt','aflat'),
-    (36,'auth::ldap_bind','ldp','aldap'),
-    (40,'badmailfrom','bmf','badmf'),
-    (41,'badmailfromto','bmt','bfrto'),
-    (42,'rhsbl','rbl','rhsbl'),
-    (44,'resolvable_fromhost','rfh','rsvfh'),
-    (45,'sender_permitted_from','spf',' spf'),
-    (50,'badrcptto','bto','badto'),
-    (51,'rcpt_map','rmp','rcmap'),
-    (52,'rcpt_regex','rcx','rcrex'),
-    (53,'qmail_deliverable','qmd',' qmd'),
-    (55,'rcpt_ok','rok','rcpok'),
-    (58,'bogus_bounce','bog','bogus'),
-    (59,'greylisting','gry','greyl'),
-    (60,'headers','hdr','headr'),
-    (61,'loop','lop','loop'),
-    (62,'uribl','uri','uribl'),
-    (63,'domainkeys','dk','dkey'),
-    (64,'dkim','dkm','dkim'),
-    (65,'spamassassin','spm','spama'),
-    (66,'dspam','dsp','dspam'),
-    (70,'virus::aveclient','vav','avirs'),
-    (71,'virus::bitdefender','vbd','bitdf'),
-    (72,'virus::clamav','cav','clamv'),
-    (73,'virus::clamdscan','cad','clamd'),
-    (74,'virus::hbedv','hbv','hbedv'),
-    (75,'virus::kavscanner','kav','kavsc'),
-    (76,'virus::klez_filter','klz','vklez'),
-    (77,'virus::sophie','sop','sophe'),
-    (78,'virus::uvscan','uvs','uvscn'),
-    (80,'queue::qmail-queue','qqm','queue'),
-    (81,'queue::maildir','qdr','qudir'),
-    (82,'queue::postfix-queue','qpf','qupfx'),
-    (83,'queue::smtp-forward','qfw','qufwd'),
-    (84,'queue::exim-bsmtp','qxm','qexim'),
-    (98,'quit_fortune','for','fortu'),
-    (99,'connection_time','tim','time');
-
-/*!40000 ALTER TABLE `plugin` ENABLE KEYS */;
-UNLOCK TABLES;
-
 
 # Dump of table plugin_aliases
 # ------------------------------------------------------------
@@ -197,33 +128,10 @@ DROP TABLE IF EXISTS `plugin_aliases`;
 
 CREATE TABLE `plugin_aliases` (
   `plugin_id` int(11) unsigned NOT NULL,
-  `name` varchar(35) character set utf8 NOT NULL default '',
-  KEY `plugin_id` (`plugin_id`),
-  CONSTRAINT `plugin_id` FOREIGN KEY (`plugin_id`) REFERENCES `plugin` (`id`) ON UPDATE CASCADE
+  `name` varchar(35) CHARACTER SET utf8 NOT NULL DEFAULT '',
+  UNIQUE KEY `plugin_id` (`plugin_id`,`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
 
-LOCK TABLES `plugin_aliases` WRITE;
-/*!40000 ALTER TABLE `plugin_aliases` DISABLE KEYS */;
-
-INSERT INTO `plugin_aliases` (`plugin_id`, `name`)
-VALUES
-    (60,'check_basicheaders'),
-    (44,'require_resolvable_fromhost'),
-    (21,'count_unrecognized_commands'),
-    (9,'check_earlytalker'),
-    (40,'check_badmailfrom'),
-    (50,'check_badrcptto'),
-    (58,'check_bogus_bounce'),
-    (15,'check_spamhelo'),
-    (3,'ident::p0f_3a0'),
-    (80,'queue::qmail_2dqueue'),
-    (22,'noop_counter');
-
-/*!40000 ALTER TABLE `plugin_aliases` ENABLE KEYS */;
-UNLOCK TABLES;
-
-
-
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
 /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
diff --git a/plugins/registry.txt b/plugins/registry.txt
index f02709c..872a239 100644
--- a/plugins/registry.txt
+++ b/plugins/registry.txt
@@ -1,13 +1,16 @@
 # This file contains a list of every plugin used on this server. If you have
 # additional plugins running, add them here.
 # Fields are whitespace delimited. Columns are ordered by numeric plugin ID.
+#
+#   the order of plugins in this file determines the order they appear in
+#   summary output
 # 
 #id	name						abb3	abb5		aliases
 #
 201	hosts_allow					alw		allow
 202	ident::geoip				geo		geoip
-203	ident::p0f					p0f		p0f
-204	ident::p0f_3a0              p0f     p0f
+203	ident::p0f					p0f		p0f			ident::p0f_3a0,ident::p0f_3a1
+
 205	karma						krm		karma
 206	dnsbl						dbl		dnsbl
 207	relay						rly		relay       check_relay,check_norelay,relay_only
@@ -26,13 +29,13 @@
 #
 # Authentication
 #
-400	auth::auth_vpopmail_sql		aut		vpsql
-401	auth::auth_vpopmaild		vpd		vpopd
-402	auth::auth_vpopmail			vpo		vpop
-403	auth::auth_checkpasswd		ckp		chkpw
-404	auth::auth_cvs_unix_local	cvs		cvsul
-405	auth::auth_flat_file		flt		aflat
-406	auth::auth_ldap_bind		ldp		aldap
+400	auth::auth_vpopmail_sql		avq		avsql
+401	auth::auth_vpopmaild		avd		vpopd
+402	auth::auth_vpopmail			avp		vpop
+403	auth::auth_checkpassword	ack		chkpw
+404	auth::auth_cvs_unix_local	acv		cvsul
+405	auth::auth_flat_file		aff		aflat
+406	auth::auth_ldap_bind		ald		aldap
 407	auth::authdeny	        	dny		adeny
 #
 # Sender / Envelope From
@@ -80,11 +83,11 @@
 #
 # Queue Plugins
 #
-800	queue::qmail-queue			qqm		queue
+800	queue::qmail-queue			qqm		queue	queue::qmail_2dqueue
 801	queue::maildir				qdr		qudir
-802	queue::postfix-queue		qpf		qupfx
-803	queue::smtp-forward			qfw		qufwd
-804	queue::exim-bsmtp			qxm		qexim
+802	queue::postfix-queue		qpf		qupfx	queue::postfix_2dqueue
+803	queue::smtp-forward			qfw		qufwd	queue::smtp_2dqueue
+804	queue::exim-bsmtp			qxm		qexim	queue::exim_2dbsmtp
 
 900	quit_fortune				for		fortu
 

From f63c029bbb90f850330837e9b60b176694115b52 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Mon, 22 Apr 2013 02:29:29 -0400
Subject: [PATCH 07/21] qmail_deliverable: smite null sender to email list

---
 plugins/karma             | 18 ++++++++++++++----
 plugins/qmail_deliverable |  6 ++++--
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/plugins/karma b/plugins/karma
index 8cc91e6..a32ed6a 100644
--- a/plugins/karma
+++ b/plugins/karma
@@ -328,8 +328,11 @@ sub rcpt_handler {
     my $recipients = scalar $self->transaction->recipients;
     return DECLINED if $recipients < 2;    # only one recipient
 
-    my $karma = $self->connection->notes('karma_history');
-    return DECLINED if $karma > 0;         # good karma, no limit
+    my $history = $self->connection->notes('karma_history');
+    return DECLINED if $history > 0;       # good history, no limit
+
+    my $karma = $self->connection->notes('karma');
+    return DECLINED if $karma > 0;      # good connection, no limit
 
     #  limit # of recipients if host has negative or unknown karma
     return $self->get_reject("too many recipients");
@@ -337,9 +340,16 @@ sub rcpt_handler {
 
 sub data_handler {
     my ($self, $transaction) = @_;
-    return DECLINED if !$self->qp->connection->relay_client;
 
-    $self->adjust_karma(5);    # big karma boost for authenticated user/IP
+    if ( $self->qp->connection->relay_client ) {
+        $self->adjust_karma(5);    # big karma boost for authenticated user/IP
+    };
+
+    my $karma = $self->connection->notes('karma');
+    if ( $karma < -3 ) {     # bad karma
+        return $self->get_reject("very bad karma: $karma");
+    };
+
     return DECLINED;
 }
 
diff --git a/plugins/qmail_deliverable b/plugins/qmail_deliverable
index 62609f8..2b31756 100644
--- a/plugins/qmail_deliverable
+++ b/plugins/qmail_deliverable
@@ -201,8 +201,10 @@ sub rcpt_handler {
     $self->log(LOGINFO, "pass, bouncesaying with program"), $k++ if $rv == 0x13;
     if ($rv == 0x14) {
         my $s = $transaction->sender->address;
-        return (DENY, "mailing lists do not accept null senders")
-          if (!$s || $s eq '<>');
+        if (!$s || $s eq '<>') {
+            $self->adjust_karma(-1);
+            return (DENY, "mailing lists do not accept null senders");
+        };
         $self->log(LOGINFO, "pass, ezmlm list");
         $k++;
     }

From 78e7a0c644a349031ac023ec853f16a0b6cff2ce Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 00:21:36 -0400
Subject: [PATCH 08/21] bump RAM from 150 to 200MB

DKIM message signing needs more RAM
---
 run | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/run b/run
index 1bbd0a6..908d775 100755
--- a/run
+++ b/run
@@ -2,8 +2,8 @@
 #
 # You might want/need to to edit these settings
 QPUSER=smtpd
-# limit qpsmtpd to 150MB memory, should be several times what is needed.
-MAXRAM=150000000
+# limit qpsmtpd to 200MB memory, should be several times what is needed.
+MAXRAM=200000000
 BIN=/usr/local/bin
 PERL=/usr/bin/perl
 

From 3d7d43e0af4ec46047cb5a93a407628b25661e27 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 00:23:24 -0400
Subject: [PATCH 09/21] split is_immune into itself + is_naughty

is_immune tests designates to plugins they should always skip processing.

That's typical for naughty connections, but this change provides the ability to handly naughty connections differently than (whitelisted/relayclients/known good) senders.
---
 lib/Qpsmtpd/Plugin.pm | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/Qpsmtpd/Plugin.pm b/lib/Qpsmtpd/Plugin.pm
index 2d3537e..4e0226f 100644
--- a/lib/Qpsmtpd/Plugin.pm
+++ b/lib/Qpsmtpd/Plugin.pm
@@ -303,6 +303,12 @@ sub is_immune {
         $self->log(LOGINFO, "skip, whitelisted sender");
         return 1;
     }
+    return;
+}
+
+sub is_naughty {
+    my $self = shift;
+
     if ($self->connection->notes('naughty')) {
 
         # see plugins/naughty
@@ -323,7 +329,7 @@ sub adjust_karma {
 
     my $karma = $self->connection->notes('karma') || 0;
     $karma += $value;
-    $self->log(LOGDEBUG, "karma adjust: $value ($karma)");
+    $self->log(LOGDEBUG, "karma $value ($karma)");
     $self->connection->notes('karma', $karma);
     return $value;
 }

From f41df6e96de38c08674b51e6494555d6e5b1dbb5 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 00:27:07 -0400
Subject: [PATCH 10/21] summarize shows a narrower screen by default.

passing in -l for when your term windows is more than 200 chars wide will show more detail
---
 log/summarize | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/log/summarize b/log/summarize
index b72cef9..539e5d3 100755
--- a/log/summarize
+++ b/log/summarize
@@ -6,9 +6,13 @@ use warnings;
 use Cwd;
 use Data::Dumper;
 use File::Tail;
+use Getopt::Std;
 
 $Data::Dumper::Sortkeys = 1;
 
+our $opt_l = 0;
+getopts('l');
+
 my (%plugins, %plugin_aliases, %seen_plugins, %pids);
 my %hide_plugins = map { $_ => 1 } qw/ hostname /;
 
@@ -32,7 +36,7 @@ my %formats = (
                ip                          => "%-15.15s",
                hostname                    => "%-20.20s",
                distance                    => "%5.5s",
-               'ident::geoip'              => "%-20.20s",
+               'ident::geoip'              => $opt_l ? "%-20.20s" : "%-6.6s",
                'ident::p0f'                => "%-10.10s",
                count_unrecognized_commands => "%-5.5s",
                unrecognized_commands       => "%-5.5s",
@@ -269,18 +273,20 @@ sub print_auto_format {
             next;
         }
 
+        my $wide = $opt_l ? 20 : 8;
+
         if (defined $pids{$pid}{helo_host} && $plugin =~ /helo/) {
-            $format .= " %-18.18s";
-            push @values, substr(delete $pids{$pid}{helo_host}, -18, 18);
+            $format .= " %-$wide.${wide}s";
+            push @values, substr(delete $pids{$pid}{helo_host}, -$wide, $wide);
             push @headers, 'HELO';
         }
         elsif (defined $pids{$pid}{from} && $plugin =~ /from/) {
-            $format .= " %-20.20s";
-            push @values, substr(delete $pids{$pid}{from}, -20, 20);
+            $format .= " %-$wide.${wide}s";
+            push @values, substr(delete $pids{$pid}{from}, -$wide, $wide);
             push @headers, 'MAIL FROM';
         }
         elsif (defined $pids{$pid}{to} && $plugin =~ /to|rcpt|recipient/) {
-            $format .= " %-20.20s";
+            $format .= " %-$wide.${wide}s";
             push @values,  delete $pids{$pid}{to};
             push @headers, 'RCPT TO';
         }
@@ -299,7 +305,7 @@ sub print_auto_format {
     $format .= "\n";
     printf("\n$format", @headers) if (!$printed || $printed % 20 == 0);
     printf($format, @values);
-    print Data::Dumper::Dumper($pids{$pid}) if keys %{$pids{$pid}};
+    #print Data::Dumper::Dumper($pids{$pid}) if keys %{$pids{$pid}};
     $printed++;
 }
 
@@ -347,6 +353,8 @@ sub populate_plugins_from_registry {
     open my $F, '<', $file;
     while (defined(my $line = <$F>)) {
         next if $line =~ /^#/;    # discard comments
+        chomp $line;
+        next if ! $line;
         my ($id, $name, $abb3, $abb5, $aliases) = split /\s+/, $line;
         next if !defined $name;
         $plugins{$name} = {id => $id, abb3 => $abb3, abb5 => $abb5};

From 88e6ce6adb1d90814e1bd6893b34c31b8e2c7a19 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 00:29:33 -0400
Subject: [PATCH 11/21] dmarc: improving and updating POD

---
 plugins/dmarc | 69 ++++++++++++++++++++++++++++++++-------------------
 1 file changed, 44 insertions(+), 25 deletions(-)

diff --git a/plugins/dmarc b/plugins/dmarc
index 95b0320..60db367 100644
--- a/plugins/dmarc
+++ b/plugins/dmarc
@@ -6,6 +6,10 @@ Domain-based Message Authentication, Reporting and Conformance
 
 =head1 SYNOPSIS
 
+DMARC is an extremely reliable means to authenticate email.
+
+=head1 DESCRIPTION
+
 From the DMARC Draft: "DMARC operates as a policy layer atop DKIM and SPF. These technologies are the building blocks of DMARC as each is widely deployed, supported by mature tools, and is readily available to both senders and receivers. They are complementary, as each is resilient to many of the failure modes of the other."
 
 DMARC provides a way to exchange authentication information and policies among mail servers.
@@ -14,10 +18,10 @@ DMARC benefits domain owners by preventing others from impersonating them. A dom
 
 DMARC benefits mail server operators by providing them with an extremely reliable (as opposed to DKIM or SPF, which both have reliability issues when used independently) means to block forged emails. Is that message really from PayPal, Chase, Gmail, or Facebook? Since those organizations, and many more, publish DMARC policies, operators have a definitive means to know.
 
-=head1 HOW IT WORKS
-
 =head1 HOWTO
 
+=head2 Protect a domain with DMARC
+
 See Section 10 of the draft: Domain Owner Actions
 
 1. Deploy DKIM & SPF
@@ -25,33 +29,47 @@ See Section 10 of the draft: Domain Owner Actions
 3. Publish a "monitor" record, ask for data reports
 4. Roll policies from monitor to reject
 
-=head2 Publish a DMARC policy
+=head3 Publish a DMARC policy
+
+_dmarc  IN TXT "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc-feedback@example.com;"
 
  v=DMARC1;    (version)
  p=none;      (disposition policy : reject, quarantine, none (monitor))
  sp=reject;   (subdomain policy: default, same as p)
- rua
  adkim=s;     (dkim alignment: s=strict, r=relaxed)
  aspf=r;      (spf  alignment: s=strict, r=relaxed)
- rua=mailto: dmarc-feedback\@$zone;  (aggregate reports)
- ruf=mailto: dmarc-feedback\@$zone.com; (forensic reports)
+ rua=mailto: dmarc-feedback@example.com; (aggregate reports)
+ ruf=mailto: dmarc-feedback@example.com; (forensic reports)
  rf=afrf;     (report format: afrf, iodef)
  ri=8400;     (report interval)
  pct=50;      (percent of messages to filter)
 
+=head2 Validate messages with DMARC
 
-=head1 DRAFT
+1. install this plugin
+
+2. install a public suffix list in config/public_suffix_list. See http://publicsuffix.org/list/
+
+3. activate this plugin (add to config/plugins)
+
+Be sure to run the DMARC after the SPF & DKIM plugins, and you should also have I<reject 0> set for both SPF and DKIM.
+
+=head2 Parse dmarc feedback reports into a database
+
+See http://www.taugh.com/rddmarc/
+
+=head1 MORE INFORMATION
 
 http://www.dmarc.org/draft-dmarc-base-00-02.txt
 
+https://github.com/qpsmtpd-dev/qpsmtpd-dev/wiki/DMARC-FAQ
+
 =head1 TODO
 
  2. provide dmarc feedback to domains that request it
 
  3. If a message has multiple 'From' recipients, reject it
 
- 4. Rejections with a 550 (perm) or 450 (temp)
-
 =head1 IMPLEMENTATION
 
 1. Primary identifier is RFC5322.From field (From: header)
@@ -99,11 +117,10 @@ sub data_post_handler {
     my $from_host = $self->get_from_host($transaction) or return DECLINED;
     my $org_host = $self->get_organizational_domain($from_host);
 
-    if (!$self->exists_in_dns($from_host)) {
-        if (!$self->exists_in_dns($org_host)) {
-            $self->log(LOGINFO, "fail, $from_host not in DNS");
-            return $self->get_reject("RFC5322.From host does not exist");
-        }
+    # 6. Receivers should reject email if the domain appears to not exist
+    if (!$self->exists_in_dns($from_host) && !$self->exists_in_dns($org_host)) {
+        $self->log(LOGINFO, "fail, $from_host not in DNS");
+        return $self->get_reject("RFC5322.From host appears non-existent");
     }
 
     # 11.2.  Determine Handling Policy
@@ -295,18 +312,20 @@ sub get_organizational_domain {
 
 sub exists_in_dns {
     my ($self, $domain) = @_;
-# the DMARC draft suggests rejecting messages whose From: domain does not
-# exist in DNS. That's as far as it goes. So I went back to the ADSP (from
-# where DMARC this originated, which in turn led me to the ietf-dkim email
-# list where a handful of 'experts' failed to agree on The Right Way to
-# perform this test. And thus no direction was given.
-# As they point out:
-#    MX records aren't mandatory.
-#    A or AAAA records as fallback aren't reliable either.
+# 6. Receivers should endeavour to reject or quarantine email if the
+#    RFC5322.From purports to be from a domain that appears to be
+#    either non-existent or incapable of receiving mail.
 
-# I chose to query the name and match NS,MX,A,or AAAA records. Since it gets
-# repeated for the for the Organizational Name, if it fails, there's no
-# delegation from the TLD.
+# I went back to the ADSP (from where DMARC this originated, which in turn
+# led me to the ietf-dkim email list where a handful of 'experts' failed to
+# agree on The Right Way to test domain validity. No direction was given.
+# They point out:
+#    MX records aren't mandatory.
+#    A or AAAA records as fallback aren't reliable.
+
+# I chose to query the From: domain name and match NS,MX,A,or AAAA records.
+# Since this search gets repeated for the Organizational Name, if it
+# fails for the O.N., there's no delegation from the TLD.
     my $res = $self->init_resolver(8);
     my $query = $res->query($domain, 'NS') or do {
         if ($res->errorstring eq 'NXDOMAIN') {

From 5aafca314fc1eac76f56aedaa308bd0d1d7c6ee5 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 03:09:02 -0400
Subject: [PATCH 12/21] SPF: arrage flow so if a pass result is possible,

we will get it and set the note for DMARC plugin
---
 plugins/dmarc                 | 26 +++++++++++++----
 plugins/sender_permitted_from | 55 ++++++++++++++---------------------
 2 files changed, 43 insertions(+), 38 deletions(-)

diff --git a/plugins/dmarc b/plugins/dmarc
index 60db367..6f41234 100644
--- a/plugins/dmarc
+++ b/plugins/dmarc
@@ -296,9 +296,16 @@ sub get_organizational_domain {
 
         # $self->log( LOGINFO, "i: $i, $tld" );
         #warn "i: $i -  tld: $tld\n";
-        if (grep /$tld/, $self->qp->config('public_suffix_list')) {
+        if (grep /^$tld/, $self->qp->config('public_suffix_list')) {
             $greatest = $i + 1;
+            next;
         }
+
+        # check for wildcards (ex: *.uk should match co.uk)
+        $tld = join '.', '\*', reverse((@labels)[0 .. $i-1]);
+        if (grep /^$tld/, $self->qp->config('public_suffix_list')) {
+            $greatest = $i + 1;
+        };
     }
 
     return $from_host if $greatest == scalar @labels;    # same
@@ -327,7 +334,16 @@ sub exists_in_dns {
 # Since this search gets repeated for the Organizational Name, if it
 # fails for the O.N., there's no delegation from the TLD.
     my $res = $self->init_resolver(8);
-    my $query = $res->query($domain, 'NS') or do {
+    return 1 if $self->host_has_rr('NS', $res, $domain);
+    return 1 if $self->host_has_rr('MX', $res, $domain);
+    return 1 if $self->host_has_rr('A',  $res, $domain);
+    return 1 if $self->host_has_rr('AAAA', $res, $domain);
+}
+
+sub host_has_rr {
+    my ($self, $type, $res, $domain) = @_;
+
+    my $query = $res->query($domain, $type) or do {
         if ($res->errorstring eq 'NXDOMAIN') {
             $self->log(LOGDEBUG, "fail, non-existent domain: $domain");
             return;
@@ -337,14 +353,14 @@ sub exists_in_dns {
     };
     my $matches = 0;
     for my $rr ($query->answer) {
-        next if $rr->type !~ /(?:NS|MX|A|AAAA)/;
+        next if $rr->type ne $type;
         $matches++;
     }
     if (0 == $matches) {
-        $self->log(LOGDEBUG, "fail, no records for $domain");
+        $self->log(LOGDEBUG, "no $type records for $domain");
     }
     return $matches;
-}
+};
 
 sub fetch_dmarc_record {
     my ($self, $zone) = @_;
diff --git a/plugins/sender_permitted_from b/plugins/sender_permitted_from
index e9a1f9e..87d418d 100644
--- a/plugins/sender_permitted_from
+++ b/plugins/sender_permitted_from
@@ -96,28 +96,17 @@ sub mail_handler {
         return (DECLINED, "SPF - null sender");
     }
 
-    if ($self->qp->connection->relay_client) {
-        $self->log(LOGINFO, "skip, relay_client");
-        return (DECLINED, "SPF - relaying permitted");
-    }
-
-    if (!$self->{_args}{reject}) {
-        $self->log(LOGINFO, "skip, reject disabled");
-        return (DECLINED);
-    }
-
-    my $client_ip = $self->qp->connection->remote_ip;
     my $from      = $sender->user . '@' . lc($sender->host);
     my $helo      = $self->qp->connection->hello_host;
     my $scope     = $from ? 'mfrom' : 'helo';
     my %req_params = (
         versions => [1, 2],    # optional
         scope => $scope,
-        ip_address => $client_ip,
+        ip_address => $self->qp->connection->remote_ip,
                      );
 
     if ($scope =~ /^mfrom|pra$/) {
-        $req_params{identity} = $from;
+        $req_params{identity}      = $from;
         $req_params{helo_identity} = $helo if $helo;
     }
     elsif ($scope eq 'helo') {
@@ -144,28 +133,24 @@ sub mail_handler {
         return (DECLINED, "SPF - no response");
     }
 
-    if (!$reject) {
-        $self->log(LOGINFO, "fail, no reject policy ($code: $why)");
-        return (DECLINED, "SPF - $code: $why");
-    }
-
-   # SPF result codes: pass fail softfail neutral none error permerror temperror
-    return $self->handle_code_none($reject, $why) if $code eq 'none';
-    if ($code eq 'fail') {
-        $self->adjust_karma(-1);
-        return $self->handle_code_fail($reject, $why);
-    }
-    elsif ($code eq 'softfail') {
-        $self->adjust_karma(-1);
-        return $self->handle_code_softfail($reject, $why);
-    }
-    elsif ($code eq 'pass') {
+    if ($code eq 'pass') {
         $self->adjust_karma(1);
         $transaction->notes('spf_pass_host', lc $sender->host);
         $self->log(LOGINFO, "pass, $code: $why");
         return (DECLINED);
     }
-    elsif ($code eq 'neutral') {
+
+    if (!$reject) {
+        $self->log(LOGINFO, "skip, tolerated ($code: $why)");
+        return (DECLINED, "SPF - $code: $why");
+    }
+
+   # SPF result codes: pass fail softfail neutral none error permerror temperror
+    return $self->handle_code_none($reject, $why) if $code eq 'none';
+    return $self->handle_code_fail($reject, $why) if $code eq 'fail';
+    return $self->handle_code_softfail($reject, $why) if $code eq 'softfail';
+
+    if ($code eq 'neutral') {
         $self->log(LOGINFO, "fail, $code, $why");
         return (DENY, "SPF - $code: $why") if $reject >= 5;
     }
@@ -196,33 +181,37 @@ sub handle_code_none {
         return (DENY, "SPF - none: $why");
     }
 
-    $self->log(LOGINFO, "pass, none, $why");
+    $self->log(LOGINFO, "skip, tolerated, none, $why");
     return DECLINED;
 }
 
 sub handle_code_fail {
     my ($self, $reject, $why) = @_;
 
+    $self->adjust_karma(-1);
+
     if ($reject >= 2) {
         $self->log(LOGINFO, "fail, $why");
         return (DENY, "SPF - forgery: $why") if $reject >= 3;
         return (DENYSOFT, "SPF - fail: $why");
     }
 
-    $self->log(LOGINFO, "pass, fail tolerated, $why");
+    $self->log(LOGINFO, "fail, tolerated, $why");
     return DECLINED;
 }
 
 sub handle_code_softfail {
     my ($self, $reject, $why) = @_;
 
+    $self->adjust_karma(-1);
+
     if ($reject >= 3) {
         $self->log(LOGINFO, "fail, soft, $why");
         return (DENY,     "SPF - fail: $why") if $reject >= 4;
         return (DENYSOFT, "SPF - fail: $why") if $reject >= 3;
     }
 
-    $self->log(LOGINFO, "pass, softfail tolerated, $why");
+    $self->log(LOGINFO, "fail, soft, tolerated, $why");
     return DECLINED;
 }
 

From b4ee9620e591033ad2170719ea6ce00540cd5a61 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:16:00 -0400
Subject: [PATCH 13/21] dmarc: added support for DMARC policy pct=NNN

---
 plugins/dmarc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/plugins/dmarc b/plugins/dmarc
index 6f41234..1c1eaa0 100644
--- a/plugins/dmarc
+++ b/plugins/dmarc
@@ -52,7 +52,7 @@ _dmarc  IN TXT "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc-feedback@example.c
 
 3. activate this plugin (add to config/plugins)
 
-Be sure to run the DMARC after the SPF & DKIM plugins, and you should also have I<reject 0> set for both SPF and DKIM.
+Be sure to run the DMARC plugin after the SPF & DKIM plugins. Configure the SPF and DKIM messages to not reject mail.
 
 =head2 Parse dmarc feedback reports into a database
 
@@ -146,6 +146,12 @@ sub data_post_handler {
     #       Domain Owner.  See Section 6.2 for details.
     return DECLINED if lc $policy->{p} eq 'none';
 
+    my $pct = $policy->{pct} || 100;
+    if ( $pct != 100 && int(rand(100)) >= $pct ) {
+        $self->log("fail, tolerated, policy, sampled out");
+        return DECLINED;
+    };
+
     return $self->get_reject("failed DMARC policy");
 }
 
@@ -348,6 +354,7 @@ sub host_has_rr {
             $self->log(LOGDEBUG, "fail, non-existent domain: $domain");
             return;
         }
+        return if $res->errorstring eq 'NOERROR';
         $self->log(LOGINFO, "error, looking up $domain: " . $res->errorstring);
         return;
     };

From 981bdf5f852c2958467d40b5fa17db571d8e291c Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:18:22 -0400
Subject: [PATCH 14/21] SPF: added more precise disposition logs, so that

postprocess can determine if a SPF failure caused a rejection
---
 plugins/sender_permitted_from | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/plugins/sender_permitted_from b/plugins/sender_permitted_from
index 87d418d..e80b4e4 100644
--- a/plugins/sender_permitted_from
+++ b/plugins/sender_permitted_from
@@ -151,22 +151,25 @@ sub mail_handler {
     return $self->handle_code_softfail($reject, $why) if $code eq 'softfail';
 
     if ($code eq 'neutral') {
-        $self->log(LOGINFO, "fail, $code, $why");
-        return (DENY, "SPF - $code: $why") if $reject >= 5;
+        if ($reject >= 5 ) {
+            $self->log(LOGINFO, "fail, $code, $why");
+            return (DENY, "SPF - $code: $why");
+        };
+        $self->log(LOGINFO, "fail, tolerated, $code, $why");
+        return (DECLINED);
     }
-    elsif ($code eq 'error') {
-        $self->log(LOGINFO, "fail, $code, $why");
+    if ($code =~ /(?:permerror|error)/ ) {
+        $self->log(LOGINFO, "fail, $code, $why") if $reject > 3;
         return (DENY,     "SPF - $code: $why") if $reject >= 6;
         return (DENYSOFT, "SPF - $code: $why") if $reject > 3;
+        $self->log(LOGINFO, "fail, tolerated, $code, $why");
+        return (DECLINED);
     }
-    elsif ($code eq 'permerror') {
-        $self->log(LOGINFO, "fail, $code, $why");
-        return (DENY,     "SPF - $code: $why") if $reject >= 6;
-        return (DENYSOFT, "SPF - $code: $why") if $reject > 3;
-    }
-    elsif ($code eq 'temperror') {
+    if ($code eq 'temperror') {
         $self->log(LOGINFO, "fail, $code, $why");
         return (DENYSOFT, "SPF - $code: $why") if $reject >= 2;
+        $self->log(LOGINFO, "fail, tolerated, $code, $why");
+        return (DECLINED);
     }
 
     $self->log(LOGINFO, "SPF from $from was $code: $why");
@@ -211,7 +214,7 @@ sub handle_code_softfail {
         return (DENYSOFT, "SPF - fail: $why") if $reject >= 3;
     }
 
-    $self->log(LOGINFO, "fail, soft, tolerated, $why");
+    $self->log(LOGINFO, "fail, tolerated, soft, $why");
     return DECLINED;
 }
 

From 6947c4fa773983be2ece5da553a39da9cfe652dc Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:20:07 -0400
Subject: [PATCH 15/21] docs/logging: added description of log prefixes

---
 STATUS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/STATUS b/STATUS
index 98050a6..6992271 100644
--- a/STATUS
+++ b/STATUS
@@ -15,6 +15,7 @@ in Perl Best Practices is also fair game.
 So far, the main changes between the release and dev branches have focused
 on these goals:
 
+  - plugins use is_immune and is_naughty instead of a local methods
   - plugins log a single entry summarizing their disposition
   - plugin logs prefixed with keywords: pass, fail, skip, error
   - plugins use 'reject' and 'reject_type' settings

From f7a59707ded511e2c8320ed5e491aa321d63e8bb Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:21:18 -0400
Subject: [PATCH 16/21] docs/logging: added description of log prefixes

---
 Changes          |  1 +
 docs/logging.pod | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/Changes b/Changes
index 74b91e2..d5b50ca 100644
--- a/Changes
+++ b/Changes
@@ -17,6 +17,7 @@
   karma: sprinkled karma awards throughout other plugins
      - limit poor karma hosts to 1 concurrent connection
      - allow +3 conncurrent connections to hosts with good karma
+     - limit recipients to 1 for senders with negative karma
 
   Sanitize spamd_sock path for perl taint mode - Markus Ullmann
 
diff --git a/docs/logging.pod b/docs/logging.pod
index 0066132..40a8747 100644
--- a/docs/logging.pod
+++ b/docs/logging.pod
@@ -127,6 +127,40 @@ plugins in plugins/logging, specifically the L<plugins/logging/warn> and
 L<plugins/logging/adaptive> files for examples of how to write your own
 logging plugins.
 
+=head1 plugin authors
+
+While plugins can log anything they like, a few logging conventions in use:
+
+=over 4
+
+=item * at LOGINFO, log a single entry summarizing their disposition
+
+=item * log messages are prefixed with keywords: pass, fail, skip, error
+
+=over 4
+
+=item pass: tests were run and the message passed
+
+=item fail: tests were run and the message failed
+
+=item fail, tolerated: tests run, msg failed, reject disabled
+
+=item skip: tests were not run
+
+=item error: tried to run tests but failure(s) encountered
+
+=item info: additional info, not to be used for plugin summary
+
+=back
+
+=item * when tests fail and reject is disabled, use the 'fail, tolerated' prefix
+
+=back
+
+When these conventions are adhered to, the logs/summarize tool outputs each
+message as a single row, with a small x showing failed tests and a large X
+for failed tests that caused message rejection.
+
 =head1 Internal support for pluggable logging
 
 Any code in the core can call C<$self->log()> and those log lines will be

From 736e3b6eb3f1c76cbe812c39e5dae31c656caf49 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:25:31 -0400
Subject: [PATCH 17/21] distinguish rejecting versus tolerated failures

---
 lib/Qpsmtpd/Plugin.pm       | 2 +-
 log/summarize               | 1 +
 plugins/helo                | 2 +-
 plugins/resolvable_fromhost | 4 ++--
 plugins/spamassassin        | 4 ++--
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/Qpsmtpd/Plugin.pm b/lib/Qpsmtpd/Plugin.pm
index 4e0226f..a72cc86 100644
--- a/lib/Qpsmtpd/Plugin.pm
+++ b/lib/Qpsmtpd/Plugin.pm
@@ -224,7 +224,7 @@ sub get_reject {
 
     my $reject = $self->{_args}{reject};
     if (defined $reject && !$reject) {
-        $self->log(LOGINFO, "fail, reject disabled" . $log_mess);
+        $self->log(LOGINFO, "fail, tolerated" . $log_mess);
         return DECLINED;
     }
 
diff --git a/log/summarize b/log/summarize
index 539e5d3..51270c3 100755
--- a/log/summarize
+++ b/log/summarize
@@ -314,6 +314,7 @@ sub show_symbol {
     return ' o' if $mess eq 'TLS setup returning';
     return ' o' if $mess eq 'pass';
     return ' -' if $mess eq 'skip';
+    return ' x' if 'fail, tolerated' eq substr($mess, 0, 15);
     return ' X' if $mess eq 'fail';
     return ' -' if $mess =~ /^skip[,:\s]/i;
     return ' o' if $mess =~ /^pass[,:\s]/i;
diff --git a/plugins/helo b/plugins/helo
index b5d7fb3..0123471 100644
--- a/plugins/helo
+++ b/plugins/helo
@@ -246,7 +246,7 @@ sub helo_handler {
     my ($self, $transaction, $host) = @_;
 
     if (!$host) {
-        $self->log(LOGINFO, "fail, no helo host");
+        $self->log(LOGINFO, "fail, tolerated, no helo host");
         return DECLINED;
     }
 
diff --git a/plugins/resolvable_fromhost b/plugins/resolvable_fromhost
index aa881a3..9804705 100644
--- a/plugins/resolvable_fromhost
+++ b/plugins/resolvable_fromhost
@@ -116,7 +116,7 @@ sub hook_mail {
             return Qpsmtpd::DSN->temp_resolver_failed($self->get_reject_type(),
                                                       '');
         }
-        $self->log(LOGINFO, 'fail, missing result, reject disabled');
+        $self->log(LOGINFO, 'fail, tolerated, missing result');
         return DECLINED;
     };
 
@@ -127,7 +127,7 @@ sub hook_mail {
 
     if (!$self->{_args}{reject}) {
         ;
-        $self->log(LOGINFO, "fail, reject disabled, $result");
+        $self->log(LOGINFO, "fail, tolerated, $result");
         return DECLINED;
     }
 
diff --git a/plugins/spamassassin b/plugins/spamassassin
index 7d7f734..342c788 100644
--- a/plugins/spamassassin
+++ b/plugins/spamassassin
@@ -178,7 +178,7 @@ sub data_post_handler {
 
     if ($transaction->data_size > 500_000) {
         $self->log(LOGINFO,
-                   "skip: too large (" . $transaction->data_size . ")");
+                   "skip, too large (" . $transaction->data_size . ")");
         return (DECLINED);
     }
 
@@ -424,7 +424,7 @@ sub reject {
 
     if ($score < $reject) {
         if ($ham_or_spam eq 'Spam') {
-            $self->log(LOGINFO, "fail, $status < $reject, $learn");
+            $self->log(LOGINFO, "fail, tolerated, $status < $reject, $learn");
             return DECLINED;
         }
         else {

From b3ca4e3ccc9cff484516c3f18d68d6ad579573d4 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:30:28 -0400
Subject: [PATCH 18/21] karma: limit rcpts to 1 for senders with neg karma

---
 plugins/karma | 45 +++++++++++++++++++++++++++------------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/plugins/karma b/plugins/karma
index a32ed6a..a8f2dd6 100644
--- a/plugins/karma
+++ b/plugins/karma
@@ -20,22 +20,22 @@ Karma provides other plugins with a karma value they can use to be more
 lenient, strict, or skip processing entirely.
 
 Karma is small, fast, and ruthlessly efficient. Karma can be used to craft
-custom connection policies such as these two examples: 
+custom connection policies such as these two examples:
 
-=over 4 
+=over 4
 
 Hi there, well known and well behaved sender. Please help yourself to greater concurrency (hosts_allow), multiple recipients (karma), and no delays (early_sender).
 
 Hi there, naughty sender. You get a max concurrency of 1, max recipients of 2, and SMTP delays.
 
-=back 
+=back
 
 =head1 CONFIG
 
 =head2 negative <integer>
 
 How negative a senders karma can get before we penalize them for sending a
-naughty message. Karma is the number of nice - naughty connections. 
+naughty message. Karma is the number of nice - naughty connections.
 
 Default: 1
 
@@ -67,7 +67,7 @@ I<0> will not reject any connections.
 
 I<1> will reject naughty senders.
 
-I<connect> is the most efficient setting. 
+I<connect> is the most efficient setting.
 
 To reject at any other connection hook, use the I<naughty> setting and the
 B<naughty> plugin.
@@ -104,7 +104,7 @@ sending a virus, early talking, or sending messages with a very high spam
 score.
 
 This plugin does not penalize connections with transaction notes I<relayclient>
-or I<whitelisthost> set. These notes would have been set by the B<relay>, 
+or I<whitelisthost> set. These notes would have been set by the B<relay>,
 B<whitelist>, and B<dns_whitelist_soft> plugins. Obviously, those plugins must
 run before B<karma> for that to work.
 
@@ -244,9 +244,10 @@ sub register {
 
     #$self->prune_db();    # keep the DB compact
     $self->register_hook('connect',       'connect_handler');
+    $self->register_hook('rcpt_pre',      'rcpt_handler');
     $self->register_hook('data',          'data_handler');
+    $self->register_hook('data_post',     'data_handler');
     $self->register_hook('disconnect',    'disconnect_handler');
-    $self->register_hook('received_line', 'rcpt_handler');
 }
 
 sub hook_pre_connection {
@@ -256,8 +257,6 @@ sub hook_pre_connection {
 
     my $remote_ip = $args{remote_ip};
 
-    #my $max_conn = $args{max_conn_ip};
-
     my $db   = $self->get_db_location();
     my $lock = $self->get_db_lock($db) or return DECLINED;
     my $tied = $self->get_db_tie($db, $lock) or return DECLINED;
@@ -323,28 +322,38 @@ sub connect_handler {
 }
 
 sub rcpt_handler {
-    my ($self, $transaction, $recipient, %args) = @_;
+    my ($self, $transaction, $addr) = @_;
 
-    my $recipients = scalar $self->transaction->recipients;
-    return DECLINED if $recipients < 2;    # only one recipient
+    return DECLINED if $self->is_immune();
+
+    my $recipients = scalar $self->transaction->recipients or do {
+        $self->log(LOGDEBUG, "info, no recipient count");
+        return DECLINED;
+    };
 
     my $history = $self->connection->notes('karma_history');
-    return DECLINED if $history > 0;       # good history, no limit
+    if ( $history > 0 ) {
+        $self->log(LOGDEBUG, "info, good history");
+        return DECLINED;
+    };
 
     my $karma = $self->connection->notes('karma');
-    return DECLINED if $karma > 0;      # good connection, no limit
+    if ( $karma > 0 ) {
+        $self->log(LOGDEBUG, "info, good connection");
+        return DECLINED;
+    };
 
     #  limit # of recipients if host has negative or unknown karma
-    return $self->get_reject("too many recipients");
+    return (DENY, "too many recipients for karma $karma (h: $history)");
 }
 
 sub data_handler {
     my ($self, $transaction) = @_;
 
-    if ( $self->qp->connection->relay_client ) {
-        $self->adjust_karma(5);    # big karma boost for authenticated user/IP
-    };
+    return DECLINED if $self->is_immune();
+    return DECLINED if $self->is_naughty();   # let naughty do it
 
+# cutting off a naughty sender at DATA prevents having to receive the message
     my $karma = $self->connection->notes('karma');
     if ( $karma < -3 ) {     # bad karma
         return $self->get_reject("very bad karma: $karma");

From e3d8a7030e4c9e767b5d4a662aa1c1341bdd5ec9 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:31:00 -0400
Subject: [PATCH 19/21] rcpt_ok: do immunity checks earlier, so that

disposition logs don't indicate failure for authenticated senders
---
 plugins/rcpt_ok | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/plugins/rcpt_ok b/plugins/rcpt_ok
index 57f64b7..7d4d201 100644
--- a/plugins/rcpt_ok
+++ b/plugins/rcpt_ok
@@ -6,18 +6,18 @@ rcpt_ok
 
 =head1 SYNOPSIS
 
-this plugin checks the standard rcpthosts config
+Validate that we accept mail for a recipient using a qmail rcpthosts file
 
 =head1 DESCRIPTION
 
-Check the recipient hostname and determine if we accept mail to that host.
+Check the envelope recipient hostname and determine if we accept mail to that host.
 
 This is functionally identical to qmail's rcpthosts implementation, consulting
 both rcpthosts and morercpthosts.cdb.
 
 =head1 CONFIGURATION
 
-It should be configured to be run _LAST_!
+It should be configured as the _LAST_ recipient plugin!
 
 =cut
 
@@ -30,6 +30,8 @@ use Qpsmtpd::DSN;
 sub hook_rcpt {
     my ($self, $transaction, $recipient, %param) = @_;
 
+    return (OK) if $self->is_immune();     # relay_client or whitelist
+
     # Allow 'no @' addresses for 'postmaster' and 'abuse'
     # qmail-smtpd will do this for all users without a domain, but we'll
     # be a bit more picky.  Maybe that's a bad idea.
@@ -37,7 +39,6 @@ sub hook_rcpt {
 
     return (OK) if $self->is_in_rcpthosts($host);
     return (OK) if $self->is_in_morercpthosts($host);
-    return (OK) if $self->qp->connection->relay_client;    # failsafe
 
     # default of relaying_denied is obviously DENY,
     # we use the default "Relaying denied" message...

From 3d6f23fcfd1bcfa21d2ebe74b3edaa45fa4635af Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:32:08 -0400
Subject: [PATCH 20/21] run: increase RAM from 200 to 300MB (dkim)

still seeing (infrequent) "too large" errors validating DKIM signatures
---
 run | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/run b/run
index 908d775..79f57ff 100755
--- a/run
+++ b/run
@@ -2,8 +2,8 @@
 #
 # You might want/need to to edit these settings
 QPUSER=smtpd
-# limit qpsmtpd to 200MB memory, should be several times what is needed.
-MAXRAM=200000000
+# limit qpsmtpd to 300MB memory
+MAXRAM=300000000
 BIN=/usr/local/bin
 PERL=/usr/bin/perl
 

From 8823de50751da52942593df0c1e347be3d902fb9 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 24 Apr 2013 16:33:57 -0400
Subject: [PATCH 21/21] dmarc test: comments in the public list was

allowing certain org domain searches to fail (plus.google.com, b/c a google.com email address was in the public list). Now I anchor the searches to the start of the line. This test also catches edge cases like co.uk, which isn't listed, but a wildcard *.uk is.
---
 t/plugin_tests/dmarc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/t/plugin_tests/dmarc b/t/plugin_tests/dmarc
index 4c8ef1c..461db72 100644
--- a/t/plugin_tests/dmarc
+++ b/t/plugin_tests/dmarc
@@ -12,7 +12,7 @@ my $test_email = 'matt@tnpi.net';
 sub register_tests {
     my $self = shift;
 
-    $self->register_test('test_get_organizational_domain', 2);
+    $self->register_test('test_get_organizational_domain', 3);
     $self->register_test("test_fetch_dmarc_record", 3);
     $self->register_test("test_discover_policy", 1);
 }
@@ -55,7 +55,8 @@ sub test_get_organizational_domain {
     my $transaction = $self->qp->transaction;
 
     cmp_ok( $self->get_organizational_domain('test.www.tnpi.net'), 'eq', 'tnpi.net' );
-    cmp_ok( $self->get_organizational_domain('www.example.co.uk'), 'eq', 'example.co.uk' )
+    cmp_ok( $self->get_organizational_domain('www.example.co.uk'), 'eq', 'example.co.uk' );
+    cmp_ok( $self->get_organizational_domain('plus.google.com'),   'eq', 'google.com' );
 };
 
 sub test_discover_policy {