From 2416d1e940369e6db1a923ee4865b2dee1dd7336 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 17 Dec 2013 23:43:26 -0500 Subject: [PATCH 01/15] Plugin.pm: made is_naughty is now a getter too --- lib/Qpsmtpd/Plugin.pm | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/Qpsmtpd/Plugin.pm b/lib/Qpsmtpd/Plugin.pm index 9693524..5dde02c 100644 --- a/lib/Qpsmtpd/Plugin.pm +++ b/lib/Qpsmtpd/Plugin.pm @@ -218,7 +218,7 @@ sub compile { sub get_reject { my $self = shift; - my $smtp_mess = shift || "why didn't you pass an error message?"; + my $smtp_mess = shift || "unspecified error"; my $log_mess = shift || ''; $log_mess = ", $log_mess" if $log_mess; @@ -320,17 +320,17 @@ sub is_immune { sub is_naughty { my ($self, $setit) = @_; - if ( defined $setit ) { - $self->connection->notes('naughty', $setit); - $self->connection->notes('rejected', $setit); - }; + # see plugins/naughty + return $self->connection->notes('naughty') if ! defined $setit; + + $self->connection->notes('naughty', $setit); + $self->connection->notes('rejected', $setit); if ($self->connection->notes('naughty')) { - - # see plugins/naughty $self->log(LOGINFO, "skip, naughty"); return 1; } + if ($self->connection->notes('rejected')) { # http://www.steve.org.uk/Software/ms-lite/ @@ -345,7 +345,7 @@ sub adjust_karma { my $karma = $self->connection->notes('karma') || 0; $karma += $value; - $self->log(LOGDEBUG, "karma $value ($karma)"); + $self->log(LOGINFO, "karma $value ($karma)"); $self->connection->notes('karma', $karma); return $value; } From 012a7a4918251b4cfa8f1c2ff992bf644354acef Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 17 Dec 2013 23:44:35 -0500 Subject: [PATCH 02/15] log/summarize: set undefined strings as empty str avoids undef warnings --- log/summarize | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/log/summarize b/log/summarize index 1c85070..6c0ad69 100755 --- a/log/summarize +++ b/log/summarize @@ -136,14 +136,14 @@ sub handle_dispatch { my ($message, $pid, $line) = @_; if ($message =~ /^dispatching MAIL FROM/i) { my ($from) = $message =~ /<(.*?)>/; - $pids{$pid}{from} = $from; + $pids{$pid}{from} = $from || ''; } elsif ($message =~ /^dispatching RCPT TO/i) { my ($to) = $message =~ /<(.*?)>/; - $pids{$pid}{to} = $to; + $pids{$pid}{to} = $to || ''; } elsif ($message =~ m/dispatching (EHLO|HELO) (.*)/) { - $pids{$pid}{helo_host} = $2; + $pids{$pid}{helo_host} = $2 || ''; } elsif ($message eq 'dispatching DATA') { } elsif ($message eq 'dispatching QUIT') { } From c202d3ef69543a5a6613a23ad1c76bd907316716 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 17 Dec 2013 23:53:00 -0500 Subject: [PATCH 03/15] dmarc integrated with Mail::DMARC reimplemented dmarc module to use Mail::DMARC updated SPF plugin to save SPF results in dmarc_spf note update dkim to store DKIM results in dkim_result & dkim_verifier notes --- plugins/dkim | 6 +- plugins/dmarc | 470 +++++++--------------------------- plugins/sender_permitted_from | 26 +- 3 files changed, 116 insertions(+), 386 deletions(-) diff --git a/plugins/dkim b/plugins/dkim index 39049dc..7351138 100644 --- a/plugins/dkim +++ b/plugins/dkim @@ -221,13 +221,14 @@ sub validate_it { $self->send_message_to_dkim($dkim, $transaction); my $result = $dkim->result; my $mess = $self->get_details($dkim); + $self->connection->notes('dkim_result', $result); + $self->connection->notes('dkim_verifier', $dkim); my $auth_str = "dkim=" .$dkim->result_detail; if ( $dkim->signature && $dkim->signature->domain ) { $auth_str .= " header.i=@" . $dkim->signature->domain; }; $self->store_auth_results( $auth_str ); - #$self->add_header($mess); foreach my $t (qw/ pass fail invalid temperror none /) { next if $t ne $result; @@ -482,7 +483,8 @@ sub send_message_to_dkim { $self->log(LOGERROR, $@) if $@; } - $dkim->CLOSE; + eval { $dkim->CLOSE; }; + $self->log(LOGERROR, $@) if $@; } sub get_policies { diff --git a/plugins/dmarc b/plugins/dmarc index cd40ec0..7e98e89 100644 --- a/plugins/dmarc +++ b/plugins/dmarc @@ -6,7 +6,7 @@ Domain-based Message Authentication, Reporting and Conformance =head1 SYNOPSIS -DMARC is an extremely reliable means to authenticate email. +DMARC is a reliable means to authenticate email. =head1 DESCRIPTION @@ -14,9 +14,9 @@ From the DMARC Draft: "DMARC operates as a policy layer atop DKIM and SPF. These DMARC provides a way to exchange authentication information and policies among mail servers. -DMARC benefits domain owners by preventing others from impersonating them. A domain owner can reliably tell other mail servers that "it it doesn't originate from this list of servers (SPF) and it is not signed (DKIM), then reject it!" DMARC also provides domain owners with a means to receive feedback and determine that their policies are working as desired. +DMARC benefits domain owners by preventing others from impersonating them. A domain owner can reliably tell other mail servers that "it it doesn't originate from this list of servers (SPF) and it is not signed (DKIM), then [ignore|quarantine|reject] it." DMARC also provides domain owners with a means to receive feedback and determine that their policies are working as desired. -DMARC benefits mail server operators by providing them with an extremely reliable (as opposed to DKIM or SPF, which both have reliability issues when used independently) means to block forged emails. Is that message really from PayPal, Chase, Gmail, or Facebook? Since those organizations, and many more, publish DMARC policies, operators have a definitive means to know. +DMARC benefits mail server operators by providing them with a more reliable (than SPF or DKIM alone) means to block forged emails. Is that message really from PayPal, Chase, Gmail, or Facebook? Since those organizations publish DMARC policies, operators have a definitive means to know. =head1 HOWTO @@ -46,26 +46,21 @@ _dmarc IN TXT "v=DMARC1; p=reject; rua=mailto:dmarc-feedback@example.com;" =head2 Validate messages with DMARC -1. install this plugin +1. install Mail::DMARC -2. install a public suffix list in config/public_suffix_list. See http://publicsuffix.org/list/ +2. install this plugin 3. activate this plugin. (add to config/plugins, listing it after SPF & DKIM. Check that SPF and DKIM are configured to not reject mail. -=head2 Parse dmarc feedback reports into a database - -See http://www.taugh.com/rddmarc/ =head1 MORE INFORMATION http://www.dmarc.org/draft-dmarc-base-00-02.txt -https://github.com/qpsmtpd-dev/qpsmtpd-dev/wiki/DMARC-FAQ +https://github.com/smtpd/qpsmtpd/wiki/DMARC-FAQ =head1 TODO - provide dmarc feedback to domains that request it - reject messages with multiple From: headers =head1 AUTHORS @@ -77,402 +72,111 @@ https://github.com/qpsmtpd-dev/qpsmtpd-dev/wiki/DMARC-FAQ use strict; use warnings; +use Data::Dumper; use Qpsmtpd::Constants; -sub init { - my ($self, $qp) = (shift, shift); - $self->{_args} = {@_}; +sub register { + my ($self, $qp, @args) = @_; + + $self->log(LOGERROR, "Bad arguments") if @args % 2; + $self->{_args} = {@args}; + $self->{_args}{reject} = 1 if !defined $self->{_args}{reject}; $self->{_args}{reject_type} ||= 'perm'; $self->{_args}{p_vals} = {map { $_ => 1 } qw/ none reject quarantine /}; -} -sub register { - my $self = shift; - - $self->register_hook('data_post', 'data_post_handler'); + eval "require Mail::DMARC::PurePerl"; + if ( $@ ) { + $self->log(LOGERROR, "failed to load Mail::DMARC::PurePerl" ); + } + else { + $self->{_dmarc} = Mail::DMARC::PurePerl->new(); + $self->register_hook('data_post', 'data_post_handler'); + }; } sub data_post_handler { my ($self, $transaction) = @_; - return DECLINED if $self->is_immune(); - - # 11.1. Extract Author Domain - my $from_dom = $self->get_from_dom($transaction) or return DECLINED; - my $org_dom = $self->get_organizational_domain($from_dom); - - # 6. Receivers should reject email if the domain appears to not exist - my $exists = $self->exists_in_dns($from_dom, $org_dom) or do { - $self->log(LOGINFO, "fail, $from_dom not in DNS"); - return $self->get_reject("RFC5322.From host appears non-existent"); + if ( $self->qp->connection->relay_client() ) { + $self->log(LOGINFO, "skip, relay client" ); + return DECLINED; # disable reporting to ourself }; - # 11.2. Determine Handling Policy - my $policy = $self->discover_policy($from_dom, $org_dom) - or return DECLINED; + my $dmarc = $self->{_dmarc}; + $dmarc->init(); + my $from = $transaction->header->get('From'); + eval { $dmarc->header_from_raw( $from ); }; + if ( $@ ) { + $self->log(LOGERROR, "unparseable from header: $from" ); + return $self->get_reject("unparseable from header"); + }; + my @recipients = $transaction->recipients; + eval { $dmarc->envelope_to( lc $recipients[0]->host ); }; # optional + eval { $dmarc->envelope_from( $transaction->sender->host ); }; # may be <> + $dmarc->spf( $transaction->notes('dmarc_spf') ); + my $dkim = $self->connection->notes('dkim_verifier'); + if ( $dkim ) { $dmarc->dkim( $dkim ); }; + $dmarc->source_ip( $self->qp->connection->remote_ip ); + eval { $dmarc->validate(); }; + if ( $@ ) { + $self->log(LOGERROR, $@ ); + return DECLINED if $self->is_immune; + $self->log(LOGINFO, "TODO: handle this validation failure"); + return DECLINED; + return $self->get_reject( $@, $@ ); + }; - # 3. Perform DKIM signature verification checks. A single email may - # contain multiple DKIM signatures. The results MUST include the - # value of the "d=" tag from all DKIM signatures that validated. - #my $dkim_sigs = $self->connection->notes('dkim_pass_domains') || []; +#$self->log(LOGINFO, "result: " . Dumper( $dmarc ) ); - # 4. Perform SPF validation checks. The results of this step - # MUST include the domain name from the RFC5321.MailFrom if SPF - # evaluation returned a "pass" result. - my $spf_dom = $transaction->notes('spf_pass_host'); + my $pol; + eval { $pol = $dmarc->result->published; }; + if ( $pol ) { + if ( $dmarc->has_valid_reporting_uri($pol->rua) ) { + eval { $dmarc->save_aggregate(); }; + $self->log(LOGERROR, $@ ) if $@; + } + else { + $self->log(LOGERROR, "has policy, no report URI" ); + }; + }; - my $effective_policy = ( $self->{_args}{is_subdomain} && defined $policy->{sp} ) - ? $policy->{sp} : $policy->{p}; + my $disposition = $dmarc->result->disposition; + my $auth_str = "dmarc=$disposition"; + $auth_str = " (p=" . $pol->p . ")" if $pol; - # 5. Conduct identifier alignment checks. - if ( $self->is_aligned($from_dom, $org_dom, $policy, $spf_dom ) ) { - $self->store_auth_results("dmarc=pass (p=$effective_policy) d=$from_dom"); + if ( $dmarc->result->result eq 'pass' ) { + $self->log(LOGINFO, "pass"); + $self->store_auth_results( $auth_str . " d=" . $dmarc->header_from); return DECLINED; }; - # 6. Apply policy. Emails that fail the DMARC mechanism check are - # disposed of in accordance with the discovered DMARC policy of the - # Domain Owner. See Section 6.2 for details. - if ( lc $effective_policy eq 'none' ) { - $self->store_auth_results("dmarc=fail (p=none) d=$from_dom"); + my $reason_type = my $comment = ''; + if ( $dmarc->result->reason && $dmarc->result->reason->[0] ) { + $reason_type = $dmarc->result->reason->[0]->type; + if ( $dmarc->result->reason->[0]->comment ) { + $comment = $dmarc->result->reason->[0]->comment; + }; + }; + if ( $disposition eq 'none' && $comment && $comment eq 'no policy') { + $self->log(LOGINFO, "skip, no policy"); return DECLINED; }; - my $pct = $policy->{pct} || 100; - if ( $pct != 100 && int(rand(100)) >= $pct ) { - $self->log("fail, tolerated, policy, sampled out"); - $self->store_auth_results("dmarc=sampled_out (p=$effective_policy) d=$from_dom"); - return DECLINED; - }; + my $log_mess = $dmarc->result->result; + $log_mess .= ", tolerated" if $disposition eq 'none'; + $log_mess .= ", $reason_type" if $reason_type; + $log_mess .= ", $comment" if $comment; + $self->log(LOGINFO, $log_mess); - $self->store_auth_results("dmarc=fail (p=$effective_policy) d=$from_dom"); + $self->store_auth_results( $auth_str . " d=" . $dmarc->header_from); + + return DECLINED if $disposition eq 'none'; + return DECLINED if ! $disposition; # for safety + return DECLINED if $self->is_immune; + + $self->adjust_karma(-3); +# at what point do we reject? return $self->get_reject("failed DMARC policy"); } -sub is_aligned { - my ($self, $from_dom, $org_dom, $policy, $spf_dom) = @_; - - # 5. Conduct identifier alignment checks. With authentication checks - # and policy discovery performed, the Mail Receiver checks if - # Authenticated Identifiers fall into alignment as decribed in - # Section 4. If one or more of the Authenticated Identifiers align - # with the RFC5322.From domain, the message is considered to pass - # the DMARC mechanism check. All other conditions (authentication - # failures, identifier mismatches) are considered to be DMARC - # mechanism check failures. - - my $dkim_sigs = $self->connection->notes('dkim_pass_domains') || []; - foreach (@$dkim_sigs) { - if ($_ eq $from_dom) { # strict alignment, requires exact match - $self->log(LOGINFO, "pass, DKIM aligned"); - $self->adjust_karma(1); - return 1; - } - next if $policy->{adkim} && lc $policy->{adkim} eq 's'; # strict pol. - # relaxed policy (default): Org. Dom must match a DKIM sig - if ( $_ eq $org_dom ) { - $self->log(LOGINFO, "pass, DKIM aligned, relaxed"); - $self->adjust_karma(1); - return 1; - }; - } - - return 0 if ! $spf_dom; - if ($spf_dom eq $from_dom) { - $self->adjust_karma(1); - $self->log(LOGINFO, "pass, SPF aligned"); - return 1; - } - return 0 if ($policy->{aspf} && lc $policy->{aspf} eq 's' ); # strict pol - if ($spf_dom eq $org_dom) { - $self->adjust_karma(1); - $self->log(LOGINFO, "pass, SPF aligned, relaxed"); - return 1; - } - - return 0; -}; - -sub discover_policy { - my ($self, $from_dom, $org_dom) = @_; - - # 1. Mail Receivers MUST query the DNS for a DMARC TXT record... - my @matches = $self->fetch_dmarc_record($from_dom, $org_dom) or return; - - # 4. Records that do not include a "v=" tag that identifies the - # current version of DMARC are discarded. - @matches = grep /v=DMARC1/i, @matches; - if (0 == scalar @matches) { - $self->log(LOGINFO, "skip, no valid record for $from_dom"); - return; - } - - # 5. If the remaining set contains multiple records, processing - # terminates and the Mail Receiver takes no action. - if (@matches > 1) { - $self->log(LOGINFO, "skip, too many records"); - return; - } - - # 6. If a retrieved policy record does not contain a valid "p" tag, or - # contains an "sp" tag that is not valid, then: - my %policy = $self->parse_policy($matches[0]); - if (!$self->has_valid_p(\%policy) || $self->has_invalid_sp(\%policy)) { - - # A. if an "rua" tag is present and contains at least one - # syntactically valid reporting URI, the Mail Receiver SHOULD - # act as if a record containing a valid "v" tag and "p=none" - # was retrieved, and continue processing; - # B. otherwise, the Mail Receiver SHOULD take no action. - my $rua = $policy{rua}; - if (!$rua || !$self->has_valid_reporting_uri($rua)) { - $self->log(LOGINFO, "skip, no valid reporting rua"); - return; - } - $policy{v} = 'DMARC1'; - $policy{p} = 'none'; - } - - return \%policy; -} - -sub has_valid_p { - my ($self, $policy) = @_; - return 1 if $self->{_args}{p_vals}{$policy}; - return 0; -} - -sub has_invalid_sp { - my ($self, $policy) = @_; - return 0 if !$self->{_args}{p_vals}{$policy}; - return 1; -} - -sub has_valid_reporting_uri { - my ($self, $rua) = @_; - return 1 if 'mailto:' eq lc substr($rua, 0, 7); - return 0; -} - -sub get_organizational_domain { - my ($self, $from_dom) = @_; - - # 1. Acquire a "public suffix" list, i.e., a list of DNS domain - # names reserved for registrations. http://publicsuffix.org/list/ - # $self->qp->config('public_suffix_list') - - # 2. Break the subject DNS domain name into a set of "n" ordered - # labels. Number these labels from right-to-left; e.g. for - # "example.com", "com" would be label 1 and "example" would be - # label 2.; - my @labels = reverse split /\./, $from_dom; - - # 3. Search the public suffix list for the name that matches the - # largest number of labels found in the subject DNS domain. Let - # that number be "x". - my $greatest = 0; - for (my $i = 0 ; $i <= scalar @labels ; $i++) { - next if !$labels[$i]; - my $tld = join '.', reverse((@labels)[0 .. $i]); - - # $self->log( LOGINFO, "i: $i, $tld" ); - #warn "i: $i - tld: $tld\n"; - if (grep /^$tld/, $self->qp->config('public_suffix_list')) { - $greatest = $i + 1; - next; - } - - # check for wildcards (ex: *.uk should match co.uk) - $tld = join '.', '\*', reverse((@labels)[0 .. $i-1]); - if (grep /^$tld/, $self->qp->config('public_suffix_list')) { - $greatest = $i + 1; - }; - } - - return $from_dom if $greatest == scalar @labels; # same - - # 4. Construct a new DNS domain name using the name that matched - # from the public suffix list and prefixing to it the "x+1"th - # label from the subject domain. This new name is the - # Organizational Domain. - return join '.', reverse((@labels)[0 .. $greatest]); -} - -sub exists_in_dns { - my ($self, $domain, $org_dom) = @_; -# 6. Receivers should endeavour to reject or quarantine email if the -# RFC5322.From purports to be from a domain that appears to be -# either non-existent or incapable of receiving mail. - -# That's all the draft says. I went back to the DKIM ADSP (which led me to -# the ietf-dkim email list where some 'experts' failed to agree on The Right -# Way to test domain validity. Let alone deliverability. They point out: -# MX records aren't mandatory, and A|AAAA as fallback aren't reliable. -# -# Some experimentation proved both cases in real world usage. Instead, I test -# existence by searching for a MX, NS, A, or AAAA record. Since this search -# is repeated for the Organizational Name, if the NS query fails, there's no -# delegation from the TLD. That's proven very reliable. - my $res = $self->init_resolver(8); - my @todo = $domain; - push @todo, $org_dom if $domain ne $org_dom; - foreach ( @todo ) { - return 1 if $self->host_has_rr('MX', $res, $_); - return 1 if $self->host_has_rr('NS', $res, $_); - return 1 if $self->host_has_rr('A', $res, $_); - return 1 if $self->host_has_rr('AAAA', $res, $_); - }; -} - -sub host_has_rr { - my ($self, $type, $res, $domain) = @_; - - my $query = $res->query($domain, $type) or do { - if ($res->errorstring eq 'NXDOMAIN') { - $self->log(LOGDEBUG, "fail, non-existent domain: $domain"); - return; - } - return if $res->errorstring eq 'NOERROR'; - $self->log(LOGINFO, "error, looking up $domain: " . $res->errorstring); - return; - }; - my $matches = 0; - for my $rr ($query->answer) { - next if $rr->type ne $type; - $matches++; - } - if (0 == $matches) { - $self->log(LOGDEBUG, "no $type records for $domain"); - } - return $matches; -}; - -sub fetch_dmarc_record { - my ($self, $zone, $org_dom) = @_; - - # 1. Mail Receivers MUST query the DNS for a DMARC TXT record at the - # DNS domain matching the one found in the RFC5322.From domain in - # the message. A possibly empty set of records is returned. - $self->{_args}{is_subdomain} = defined $org_dom ? 0 : 1; - my $res = $self->init_resolver(); - my $query = $res->send('_dmarc.' . $zone, 'TXT'); - my @matches; - for my $rr ($query->answer) { - next if $rr->type ne 'TXT'; - - # 2. Records that do not start with a "v=" tag that identifies the - # current version of DMARC are discarded. - next if 'v=' ne lc substr($rr->txtdata, 0, 2); - next if 'v=spf' eq lc substr($rr->txtdata, 0, 5); # SPF commonly found - $self->log(LOGINFO, $rr->txtdata); - push @matches, join('', $rr->txtdata); - } - return @matches if scalar @matches; # found one! (at least) - - # 3. If the set is now empty, the Mail Receiver MUST query the DNS for - # a DMARC TXT record at the DNS domain matching the Organizational - # Domain in place of the RFC5322.From domain in the message (if - # different). This record can contain policy to be asserted for - # subdomains of the Organizational Domain. - if ( defined $org_dom ) { # <- recursion break - if ( $org_dom eq $zone ) { - $self->log(LOGINFO, "skip, no policy for $zone (same org)"); - return @matches; - }; - return $self->fetch_dmarc_record($org_dom); # <- recursion - }; - - $self->log(LOGINFO, "skip, no policy for $zone"); - return @matches; -} - -sub get_from_dom { - my ($self, $transaction) = @_; - - my $from = $transaction->header->get('From') or do { - $self->log(LOGINFO, "error, unable to retrieve From header!"); - return; - }; - my ($from_dom) = (split /@/, $from)[-1]; # grab everything after the @ - ($from_dom) = split /\s+/, $from_dom; # remove any trailing cruft - chomp $from_dom; # remove \n - chop $from_dom if '>' eq substr($from_dom, -1, 1); # remove closing > - $self->log(LOGDEBUG, "info, from_dom is $from_dom"); - return $from_dom; -} - -sub parse_policy { - my ($self, $str) = @_; - $str =~ s/\s//g; # remove all whitespace - my %dmarc = map { split /=/, $_ } split /;/, $str; - - #warn Data::Dumper::Dumper(\%dmarc); - return %dmarc; -} - -sub external_report { - -=pod - -The report SHOULD include the following data: - - o Enough information for the report consumer to re-calculate DMARC - disposition based on the published policy, message dispositon, and - SPF, DKIM, and identifier alignment results. {R12} - - o Data for each sender subdomain separately from mail from the - sender's organizational domain, even if no subdomain policy is - applied. {R13} - - o Sending and receiving domains {R17} - - o The policy requested by the Domain Owner and the policy actually - applied (if different) {R18} - - o The number of successful authentications {R19} - - o The counts of messages based on all messages received even if - their delivery is ultimately blocked by other filtering agents {R20} - -=cut - -}; - -sub verify_external_reporting { - -=head2 Verify External Destinations - - 1. Extract the host portion of the authority component of the URI. - Call this the "destination host". - - 2. Prepend the string "_report._dmarc". - - 3. Prepend the domain name from which the policy was retrieved. - - 4. Query the DNS for a TXT record at the constructed name. If the - result of this request is a temporary DNS error of some kind - (e.g., a timeout), the Mail Receiver MAY elect to temporarily - fail the delivery so the verification test can be repeated later. - - 5. If the result includes no TXT resource records or multiple TXT - resource records, a positive determination of the external - reporting relationship cannot be made; stop. - - 6. Parse the result, if any, as a series of "tag=value" pairs, i.e., - the same overall format as the policy record. In particular, the - "v=DMARC1" tag is mandatory and MUST appear first in the list. - If at least that tag is present and the record overall is - syntactically valid per Section 6.3, then the external reporting - arrangement was authorized by the destination ADMD. - - 7. If a "rua" or "ruf" tag is thus discovered, replace the - corresponding value extracted from the domain's DMARC policy - record with the one found in this record. This permits the - report receiver to override the report destination. However, to - prevent loops or indirect abuse, the overriding URI MUST use the - same destination host from the first step. - -=cut - -} diff --git a/plugins/sender_permitted_from b/plugins/sender_permitted_from index 1f16a8d..7b049a9 100644 --- a/plugins/sender_permitted_from +++ b/plugins/sender_permitted_from @@ -53,6 +53,7 @@ The reject options are modeled after, and aim to match the functionality of thos =head1 AUTHOR + Matt Simerson - 2013 - populate dmarc_spf note with SPF results Matt Simerson - 2012 - increased policy options from 3 to 6 Matt Simerson - 2011 - rewrote using Mail::SPF Matt Sergeant - 2003 - initial plugin @@ -88,11 +89,22 @@ sub register { sub mail_handler { my ($self, $transaction, $sender, %param) = @_; - return (DECLINED) if $self->is_immune(); + if ( $self->is_immune() ) { + $transaction->notes('dmarc_spf', { + domain => $sender->host, + scope => 'mfrom', + result => 'pass', + } ); + return (DECLINED); + }; my $format = $sender->format; if ($format eq '<>' || !$sender->host || !$sender->user) { $self->log(LOGINFO, "skip, null sender"); + $transaction->notes('dmarc_spf', { + scope => 'helo', + result => 'none', + } ); return (DECLINED, "SPF - null sender"); } @@ -114,6 +126,12 @@ sub mail_handler { $req_params{helo_identity} = $helo; } + $transaction->notes('dmarc_spf', { + domain => $scope eq 'helo' ? $helo : $sender->host, + scope => $scope, + result => 'none', + } ); + my $spf_server = Mail::SPF::Server->new(); my $request = Mail::SPF::Request->new(%req_params); my $result = $spf_server->process($request) or do { @@ -133,6 +151,12 @@ sub mail_handler { return (DECLINED, "SPF - no response"); } + $transaction->notes('dmarc_spf', { + domain => $scope eq 'helo' ? $helo : $sender->host, + scope => $scope, + result => $code, + } ); + $self->store_auth_results("spf=$code smtp.mailfrom=".$sender->host); if ($code eq 'pass') { From 7a855d4d6bbcd75d834a1a1d9990ebbd77df564c Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 17 Dec 2013 23:56:02 -0500 Subject: [PATCH 04/15] added dmarc (run SPF & DKIM) first comment --- config.sample/plugins | 1 + 1 file changed, 1 insertion(+) diff --git a/config.sample/plugins b/config.sample/plugins index 46e75d6..28684a6 100644 --- a/config.sample/plugins +++ b/config.sample/plugins @@ -73,6 +73,7 @@ headers reject 0 reject_type temp require From,Date future 2 past 15 bogus_bounce log #loop dkim reject 0 +# dmarc requires dkim and SPF to run before it dmarc # content filters From 725a8d1960c5b2310aa8a054a2ab2b99d44939a6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 17 Dec 2013 23:59:57 -0500 Subject: [PATCH 05/15] dspam: remove hard coded default in train_ methods --- plugins/dspam | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/plugins/dspam b/plugins/dspam index e9f8be6..2b6a8b5 100644 --- a/plugins/dspam +++ b/plugins/dspam @@ -208,7 +208,8 @@ use IO::Handle; use Socket qw(:DEFAULT :crlf); sub register { - my ($self, $qp) = (shift, shift); + my $self = shift; + my $qp = shift; $self->log(LOGERROR, "Bad parameters for the dspam plugin") if @_ % 2; @@ -590,20 +591,8 @@ sub train_error_as_ham { my $dspam_bin = $self->{_args}{dspam_bin} || '/usr/local/bin/dspam'; my $cmd = "$dspam_bin --user $user --mode=toe --source=error --class=innocent --deliver=summary --stdout"; - my $response = $self->dspam_process($cmd, $transaction); - if ($response) { - $transaction->notes('dspam', $response); - } - else { - $transaction->notes( - 'dspam', - { - class => 'Innocent', - result => 'Innocent', - confidence => 1 - } - ); - } + $self->dspam_process($cmd, $transaction); + return; } sub train_error_as_spam { @@ -614,20 +603,8 @@ sub train_error_as_spam { my $dspam_bin = $self->{_args}{dspam_bin} || '/usr/local/bin/dspam'; my $cmd = "$dspam_bin --user $user --mode=toe --source=error --class=spam --deliver=summary --stdout"; - my $response = $self->dspam_process($cmd, $transaction); - if ($response) { - $transaction->notes('dspam', $response); - } - else { - $transaction->notes( - 'dspam', - { - class => 'Spam', - result => 'Spam', - confidence => 1 - } - ); - } + $self->dspam_process($cmd, $transaction); + return; } sub autolearn { @@ -649,6 +626,7 @@ sub autolearn { $self->autolearn_naughty($response, $transaction) and return; $self->autolearn_karma($response, $transaction) and return; $self->autolearn_spamassassin($response, $transaction) and return; + return; } sub autolearn_naughty { @@ -723,8 +701,9 @@ sub autolearn_spamassassin { $self->log(LOGINFO, "training SA FN as spam"); $self->train_error_as_spam($transaction); return 1; - } - elsif ( $sa->{is_spam} eq 'No' + }; + + if ( $sa->{is_spam} eq 'No' && $sa->{autolearn} eq 'ham' && $response->{result} eq 'Spam') { From 96dfb08d8738968f6c2dafe791394533efe979a4 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:00:52 -0500 Subject: [PATCH 06/15] headers: added POD descripting each header --- plugins/headers | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/plugins/headers b/plugins/headers index 1465e67..4bdd275 100644 --- a/plugins/headers +++ b/plugins/headers @@ -218,3 +218,79 @@ sub invalid_date_range { return; } +__END__ + +=head1 SMTP HEADERS + +http://forum.unifiedemail.net/default.aspx?g=posts&t=68 + +=head2 From: + +The eMail address, and optionally the name of the author(s). In many eMail clients not changeable except through changing account settings. + +=head2 To: + +The eMail address(es), and optionally name(s) of the message's recipient(s). Indicates primary recipients (multiple allowed), for secondary recipients see Cc: and Bcc: below. + +=head2 Subject: + +A brief summary of the topic of the message. Certain abbreviations are commonly used in the subject, including "RE:" and "FW:". + +=head2 Date: + +The local time and date when the message was written. Like the From: field, many email clients fill this in automatically when sending. The recipient's client may then display the time in the format and time zone local to him/her. + +=head2 Message-ID: + +Also an automatically generated field; used to prevent multiple delivery and for reference in In-Reply-To: (see below). + +=head2 Bcc: + +Blind Carbon Copy; addresses added to the SMTP delivery list but not (usually) listed in the message data, remaining invisible to other recipients. + +=head2 Cc: + +Carbon copy; Many eMail clients will mark eMail in your inbox differently depending on whether you are in the To: or Cc: list. + +=head2 Content-Type: + +Information about how the message is to be displayed, usually a MIME type. + +=head2 In-Reply-To: + +Message-ID of the message that this is a reply to. Used to link related messages together. + +=head2 Precedence: + +Commonly with values "bulk", "junk", or "list"; used to indicate that automated "vacation" or "out of office" responses should not be returned for this mail, e.g. to prevent vacation notices from being sent to all other subscribers of a mailinglist. + +=head2 Received: + +Tracking information generated by mail servers that have previously handled a message, in reverse order (last handler first). + +=head2 References: + +Message-ID of the message that this is a reply to, and the message-id of the message the previous was reply a reply to, etc. + +=head2 Reply-To: + +Address that should be used to reply to the message. + +=head2 Sender: + +Address of the actual sender acting on behalf of the author listed in the From: field (secretary, list manager, etc.). + +=head2 Return-Path: + +When the delivery SMTP server makes the "final delivery" of a message, it inserts a return-path line at the beginning of the mail data. Thisuse of return-path is required; mail systems MUST support it. The return-path line preserves the information in the from the MAIL command. + +=head2 Error-To: + +Indicates where error messages should be sent. In the absence of this line, they go to the Sender:, and absent that, the From: address. + +=head2 X-* + +No standard header field will ever begin with the characters "X-", so application developers are free to use them for their own purposes. + +=cut + From a4695cec8bd22e436a0c65d9c3f60bf4f1434f0f Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:02:07 -0500 Subject: [PATCH 07/15] geoip: added named array for invalid args so it passes Perl::Critic tests --- plugins/ident/geoip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/ident/geoip b/plugins/ident/geoip index b25408b..b24c460 100644 --- a/plugins/ident/geoip +++ b/plugins/ident/geoip @@ -116,10 +116,10 @@ use Qpsmtpd::Constants; #use Math::Trig; # eval'ed in set_distance_gc sub register { - my ($self, $qp) = shift, shift; + my ($self, $qp, @args) = @_; - $self->log(LOGERROR, "Bad arguments") if @_ % 2; - $self->{_args} = {@_}; + $self->log(LOGERROR, "Bad arguments") if @args % 2; + $self->{_args} = {@args}; $self->{_args}{db_dir} ||= '/usr/local/share/GeoIP'; eval 'use Geo::IP'; From a19b7de7871e83e0af25b7be3a0362db9c56e47e Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:09:10 -0500 Subject: [PATCH 08/15] updated DMARC plugin tests disabled for now, b/c they tested methods which no longer exist in new plugin --- t/plugin_tests/dmarc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/t/plugin_tests/dmarc b/t/plugin_tests/dmarc index 461db72..093c3a7 100644 --- a/t/plugin_tests/dmarc +++ b/t/plugin_tests/dmarc @@ -12,9 +12,7 @@ my $test_email = 'matt@tnpi.net'; sub register_tests { my $self = shift; - $self->register_test('test_get_organizational_domain', 3); - $self->register_test("test_fetch_dmarc_record", 3); - $self->register_test("test_discover_policy", 1); +# TODO: test against newer DMARC plugin that uses Mail::DMARC } sub setup_test_headers { From fd4cc6f8abfadc78f08f219d2c89cc5f30b9e9b6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:11:16 -0500 Subject: [PATCH 09/15] Qpsmtpd: version bump to 0.94 --- lib/Qpsmtpd.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Qpsmtpd.pm b/lib/Qpsmtpd.pm index fc41789..9e3d3e2 100644 --- a/lib/Qpsmtpd.pm +++ b/lib/Qpsmtpd.pm @@ -7,7 +7,7 @@ use Qpsmtpd::Constants; #use DashProfiler; -$VERSION = "0.93"; +$VERSION = "0.94"; my $git; From 9f88e374c2b4908732e6cffdae84fb7f3b3e386b Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:11:53 -0500 Subject: [PATCH 10/15] tls: reduced importants of an info message from WARN to INFO --- plugins/tls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/tls b/plugins/tls index 4aceaad..5d94565 100644 --- a/plugins/tls +++ b/plugins/tls @@ -141,7 +141,7 @@ sub hook_unrecognized_command { return DENY, "TLS Negotiation Failed"; } - $self->log(LOGWARN, "TLS setup returning"); + $self->log(LOGINFO, "TLS setup returning"); return DONE; } From 02da55e06d94a3b8abb3aed13d4f59fca81fbbf2 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:15:20 -0500 Subject: [PATCH 11/15] karma: added penalty for spammy TLDs --- plugins/karma | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/plugins/karma b/plugins/karma index 4dd0437..da1d515 100644 --- a/plugins/karma +++ b/plugins/karma @@ -210,8 +210,14 @@ There is little to be gained by listing servers that are already on DNS blacklists, send to invalid users, earlytalkers, etc. Those already have very lightweight tests. +=head1 TODO + + * Avoid storing results for DNSBL listed IPs + * some type of ASN integration, for tracking karma of 'neighborhoods' + =head1 AUTHOR + 2013 - MS - Addeded penalty for spammy TLDs 2012 - Matt Simerson - msimerson@cpan.org =head1 ACKNOWLEDGEMENTS @@ -244,8 +250,8 @@ sub register { #$self->prune_db(); # keep the DB compact $self->register_hook('connect', 'connect_handler'); - $self->register_hook('mail_pre', 'from_handler'); - $self->register_hook('rcpt_pre', 'rcpt_handler'); + $self->register_hook('mail', 'from_handler'); + $self->register_hook('rcpt', 'rcpt_handler'); $self->register_hook('data', 'data_handler'); $self->register_hook('data_post', 'data_handler'); $self->register_hook('disconnect', 'disconnect_handler'); @@ -323,17 +329,32 @@ sub connect_handler { } sub from_handler { - my ($self, $transaction, $addr) = @_; + my ($self,$transaction, $sender, %args) = @_; # test if sender has placed an illegal (RFC (2)821) space in envelope from my $full_from = $self->connection->notes('envelope_from'); $self->illegal_envelope_format( $full_from ); + my %spammy_tlds = ( + map { $_ => 4 } qw/ info pw /, + map { $_ => 3 } qw/ tw biz /, + map { $_ => 2 } qw/ cl br fr be jp no se sg /, + ); + foreach my $tld ( keys %spammy_tlds ) { + my $len = length $tld; + my $score = $spammy_tlds{$tld} or next; + $len ++; + if ( $sender->host && ".$tld" eq substr($sender->host,-$len,$len) ) { + $self->log(LOGINFO, "penalizing .$tld envelope sender"); + $self->adjust_karma(-$score); + }; + }; + return DECLINED; }; sub rcpt_handler { - my ($self, $transaction, $addr) = @_; + my ($self,$transaction, $recipient, %args) = @_; $self->illegal_envelope_format( $self->connection->notes('envelope_rcpt'), @@ -342,7 +363,7 @@ sub rcpt_handler { my $count = $self->connection->notes('recipient_count') || 0; $count++; if ( $count > 1 ) { - $self->log(LOGINFO, "recipients c: $count ($addr)"); + $self->log(LOGINFO, "recipients c: $count ($recipient)"); $self->connection->notes('recipient_count', $count); }; @@ -352,7 +373,7 @@ sub rcpt_handler { $self->log(LOGDEBUG, "info, no recipient count"); return DECLINED; }; - $self->log(LOGINFO, "recipients t: $recipients ($addr)"); + $self->log(LOGINFO, "recipients t: $recipients ($recipient)"); my $history = $self->connection->notes('karma_history'); if ( $history > 0 ) { @@ -378,7 +399,7 @@ sub data_handler { # cutting off a naughty sender at DATA prevents having to receive the message my $karma = $self->connection->notes('karma'); - if ( $karma < -3 ) { # bad karma + if ( $karma < -4 ) { # bad karma return $self->get_reject("very bad karma: $karma"); }; @@ -403,7 +424,7 @@ sub disconnect_handler { my $history = ($nice || 0) - $naughty; my $log_mess = ''; - if ($karma < -1) { # they achieved at least 2 strikes + if ($karma < -2) { # they achieved at least 2 strikes $history--; my $negative_limit = 0 - $self->{_args}{negative}; if ($history <= $negative_limit) { @@ -420,7 +441,7 @@ sub disconnect_handler { $log_mess = "negative"; } } - elsif ($karma > 1) { + elsif ($karma > 2) { $nice++; $log_mess = "positive"; } @@ -439,7 +460,7 @@ sub illegal_envelope_format { # test if envelope address has an illegal (RFC (2)821) space if ( uc substr($addr,0,6) ne 'FROM:<' && uc substr($addr,0,4) ne 'TO:<' ) { $self->log(LOGINFO, "illegal envelope address format: $addr" ); - $self->adjust_karma(-1); + $self->adjust_karma(-2); }; }; From bcc6adae1931c38d6b652a077d216efb2250b328 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:16:02 -0500 Subject: [PATCH 12/15] helo: add karma penalty for no HELO hostname --- plugins/helo | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/helo b/plugins/helo index d6ab0b5..500fe79 100644 --- a/plugins/helo +++ b/plugins/helo @@ -253,6 +253,7 @@ sub helo_handler { if (!$host) { $self->log(LOGINFO, "fail, tolerated, no helo host"); + $self->adjust_karma(-2); return DECLINED; } From 45316487e3c2e383b11e42af12e84199561a6790 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:16:41 -0500 Subject: [PATCH 13/15] anglebrackets: increase penalty, prefix log msgs --- plugins/dont_require_anglebrackets | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/dont_require_anglebrackets b/plugins/dont_require_anglebrackets index c8f25fd..16841e9 100644 --- a/plugins/dont_require_anglebrackets +++ b/plugins/dont_require_anglebrackets @@ -24,9 +24,9 @@ MAIL FROM:user@example.com sub hook_mail_pre { my ($self, $transaction, $addr) = @_; unless ($addr =~ /^<.*>$/) { - $self->log(LOGINFO, "added MAIL angle brackets"); $addr = '<' . $addr . '>'; - $self->adjust_karma(-1); + $self->adjust_karma(-2); + $self->log(LOGINFO, "fail, added MAIL angle brackets"); } return (OK, $addr); } @@ -34,9 +34,9 @@ sub hook_mail_pre { sub hook_rcpt_pre { my ($self, $transaction, $addr) = @_; unless ($addr =~ /^<.*>$/) { - $self->log(LOGINFO, "added RCPT angle brackets"); $addr = '<' . $addr . '>'; - $self->adjust_karma(-1); + $self->adjust_karma(-2); + $self->log(LOGINFO, "fail, added RCPT angle brackets"); } return (OK, $addr); } From 2d4f4a299a6470b0bc59cb1915f5f73e40bfcf5c Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:17:37 -0500 Subject: [PATCH 14/15] naughty: legibility improvement --- plugins/naughty | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/naughty b/plugins/naughty index 3b41826..caea455 100644 --- a/plugins/naughty +++ b/plugins/naughty @@ -140,11 +140,8 @@ sub naughty { return DECLINED; }; $self->log(LOGINFO, "disconnecting"); - my $type = $self->get_reject_type( - 'disconnect', - $self->connection->notes( - 'naughty_reject_type') - ); + my $rtype = $self->connection->notes( 'naughty_reject_type' ); + my $type = $self->get_reject_type( 'disconnect', $rtype ); return ($type, $naughty); } From 0e0cda6d95d2208e5050ebe1a9d663a374c31ddd Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 00:21:46 -0500 Subject: [PATCH 15/15] updated Changes with some 0.94 commits --- Changes | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Changes b/Changes index 3e377a8..ae058b3 100644 --- a/Changes +++ b/Changes @@ -1,4 +1,15 @@ +0.94 ___ NN, 2014 + + Updated DMARC plugin to use Mail::DMARC + + Updated SPF & DKIM plugins to store data for DMARC processing + + karma plugin: added spammy TLD penalty + + a few more log prefixes (corralling stragglers) + + 0.93 Dec 17, 2013 Added Authentication-Results header