byterazor/qpsmtpd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #139 from msimerson/geoip

Browse Source 
geoip: add GeoIP2 and ASN support
This commit is contained in:
Jared Johnson 2014-11-06 11:53:04 -06:00
commit cd79bff78e
4 changed files with 223 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile.PL
View File

@ -30,8 +30,9 @@ WriteMakefile(
# 'DBIx::Simple' => 0, # log2sql
# modules that cause Travis build tests to fail
# 'Mail::SpamAssassin' => 0,
# 'Geo::IP' => 0,
# 'Math::Complex' => 0, # geodesic distance in Geo::IP
# 'GeoIP2' => 2,
# 'Geo::IP' => 1,
# 'Math::Complex' => 0, # geodesic distance in geoip
# 'Mail::SPF' => 0,
},
ABSTRACT => 'Flexible smtpd daemon written in Perl',

6
lib/Qpsmtpd/Base.pm
View File

@ -39,4 +39,10 @@ sub is_valid_ip {
return;
}
sub is_ipv6 {
my ($self, $ip) = @_;
return if !$ip;
return Net::IP::ip_is_ipv6($ip);
};
1;

193
plugins/ident/geoip
View File

@ -6,8 +6,8 @@ geoip - provide geographic information about mail senders.
=head1 SYNOPSIS
Use MaxMind's GeoIP databases and the Geo::IP perl module to report geographic
information about incoming connections.
Use MaxMind's GeoIP databases and the GeoIP2 or Geo::IP perl modules to report
geographic information about incoming connections.
=head1 DESCRIPTION
@ -85,6 +85,8 @@ This plugin does not update the GeoIP databases. You may want to.
=head1 CHANGES
2014-06 - Matt Simerson - added GeoIP2 support
2012-06 - Matt Simerson - added GeoIP City support, continent, distance
2012-05 - Matt Simerson - added geoip_country_name note, added tests
@ -100,6 +102,8 @@ data source: http://software77.net/geo-ip/
=head1 ACKNOWLEDGEMENTS
MaxMind - the packager and distributor of the free GeoIP data
Stevan Bajic, the DSPAM author, who suggested SNARE, which describes using
geodesic distance to determine spam probability. The research paper on SNARE
can be found here:
@ -110,9 +114,11 @@ http://smartech.gatech.edu/bitstream/handle/1853/25135/GT-CSE-08-02.pdf
use strict;
use warnings;
use lib 'lib';
use Qpsmtpd::Constants;
#use Geo::IP; # eval'ed in register()
#use GeoIP2; # eval'ed in register()
#use Geo::IP; # eval loaded if GeoIP2 doesn't
#use Math::Trig; # eval'ed in set_distance_gc
sub register {
@ -122,6 +128,13 @@ sub register {
$self->{_args} = {@args};
$self->{_args}{db_dir} ||= '/usr/local/share/GeoIP';
$self->load_geoip2() and return;
$self->load_geoip1();
}
sub load_geoip1 {
my $self = shift;
eval 'use Geo::IP';
if ($@) {
warn "could not load Geo::IP";
@ -129,18 +142,84 @@ sub register {
return;
}
$self->open_geoip_db();
# Note that opening the GeoIP DB only in register has caused problems before:
# https://github.com/smtpd/qpsmtpd/commit/29ea9516806e9a8ca6519fcf987dbd684793ebdd#plugins/ident/geoip
# Opening the DB anew for every connection is horribly inefficient.
# Instead, attempt to reopen upon connect if the DB connection fails.
$self->open_geoip_db();
$self->init_my_country_code();
$self->register_hook('connect', 'connect_handler');
$self->register_hook('connect', 'geoip_lookup');
}
sub connect_handler {
sub load_geoip2 {
my $self = shift;
eval 'use GeoIP2::Database::Reader';
if ($@) {
$self->log(LOGERROR, "could not load GeoIP2");
return;
}
$self->log(LOGINFO, "GeoIP2 loaded");
eval {
$self->{_geoip2_city} = GeoIP2::Database::Reader->new(
file => $self->{_args}{db_dir} . '/GeoLite2-City.mmdb',
);
};
if ($@) {
$self->log(LOGERROR, "unable to load GeoLite2-City.mmdb");
}
eval {
$self->{_geoip2_country} = GeoIP2::Database::Reader->new(
file => $self->{_args}{db_dir} . '/GeoLite2-Country.mmdb',
);
};
if ($@) {
$self->log(LOGERROR, "unable to load GeoLite2-Country.mmdb");
}
if ($self->{_geoip2_city} || $self->{_geoip2_country}) {
$self->register_hook('connect', 'geoip2_lookup');
return 1;
}
return;
}
sub geoip2_lookup {
my $self = shift;
my $ip = $self->qp->connection->remote_ip;
if ($self->{_geoip2_city}) {
my $city_rec = $self->{_geoip2_city}->city(ip => $ip);
if ($city_rec) {
$self->qp->connection->notes('geoip_country', $city_rec->country->iso_code());
$self->qp->connection->notes('geoip_country_name', $city_rec->country->name());
$self->qp->connection->notes('geoip_continent', $city_rec->continent->code());
$self->qp->connection->notes('geoip_city', $city_rec->city->name());
$self->qp->connection->notes('geoip_asn', $city_rec->traits->autonomous_system_number());
return DECLINED;
}
}
if ($self->{_geoip2_country}) {
my $country_rec = $self->{_geoip2_country}->country(ip => $ip);
if ($country_rec) {
$self->qp->connection->notes('geoip_country', $country_rec->country->iso_code());
$self->qp->connection->notes('geoip_country_name', $country_rec->country->name());
$self->qp->connection->notes('geoip_continent', $country_rec->continent->code());
};
}
return DECLINED;
}
sub geoip_lookup {
my $self = shift;
# reopen the DB if Geo::IP failed due to DB update
@ -150,7 +229,8 @@ sub connect_handler {
$self->log(LOGINFO, "skip, no results");
return DECLINED;
};
$self->qp->connection->notes('geoip_country', $c_code);
$self->set_asn();
my $c_name = $self->set_country_name();
my ($city, $continent_code, $distance) = '';
@ -162,8 +242,9 @@ sub connect_handler {
}
my @msg_parts;
push @msg_parts, $continent_code
if $continent_code && $continent_code ne '--';
if ($continent_code && $continent_code ne '--') {
push @msg_parts, $continent_code;
};
push @msg_parts, $c_code if $c_code;
#push @msg_parts, $c_name if $c_name;
@ -190,10 +271,19 @@ sub open_geoip_db {
# save the handles in different locations
my $db_dir = $self->{_args}{db_dir};
foreach my $db (qw/ GeoIPCity GeoLiteCity /) {
if (-f "$db_dir/$db.dat") {
$self->log(LOGDEBUG, "using db $db");
$self->{_geoip_city} = Geo::IP->open("$db_dir/$db.dat");
}
next if !-f "$db_dir/$db.dat";
$self->log(LOGINFO, "using db $db");
$self->{_geoip_city} = Geo::IP->open("$db_dir/$db.dat");
}
if (-f "$db_dir/GeoIPASNum.dat") {
$self->log(LOGINFO, "using GeoIPASNum");
$self->{GeoIPASNum} = Geo::IP->open("$db_dir/GeoIPASNum.dat");
}
if (-f "$db_dir/GeoIPASNumv6.dat") {
$self->log(LOGINFO, "using GeoIPASNumv6");
$self->{GeoIPASNumv6} = Geo::IP->open("$db_dir/GeoIPASNumv6.dat");
}
# can't think of a good reason to load country if city data is present
@ -211,9 +301,13 @@ sub init_my_country_code {
sub set_country_code {
my $self = shift;
return $self->get_country_code_gc() if $self->{_geoip_city};
my $remote_ip = $self->qp->connection->remote_ip;
my $code = $self->get_country_code();
my $code = $self->{_geoip_city}
? $self->get_country_code_gc($remote_ip)
: $self->get_country_code($remote_ip);
return if ! $code;
$self->qp->connection->notes('geoip_country', $code);
return $code;
}
@ -221,7 +315,9 @@ sub set_country_code {
sub get_country_code {
my $self = shift;
my $ip = shift || $self->qp->connection->remote_ip;
return $self->get_country_code_gc($ip) if $self->{_geoip_city};
if ($self->{_geoip_city}) {
return $self->get_country_code_gc($ip);
};
return $self->{_geoip}->country_code_by_addr($ip);
}
@ -235,44 +331,73 @@ sub get_country_code_gc {
sub set_country_name {
my $self = shift;
return $self->set_country_name_gc() if $self->{_geoip_city};
my $remote_ip = $self->qp->connection->remote_ip;
my $name = $self->{_geoip}->country_name_by_addr($remote_ip) or return;
my $name = $self->{_geoip_city}
? $self->get_country_name_gc($remote_ip)
: $self->{_geoip}->country_name_by_addr($remote_ip);
return if ! $name;
$self->qp->connection->notes('geoip_country_name', $name);
return $name;
}
sub set_country_name_gc {
sub get_country_name_gc {
my $self = shift;
return if !$self->{_geoip_record};
my $remote_ip = $self->qp->connection->remote_ip;
my $name = $self->{_geoip_record}->country_name() or return;
$self->qp->connection->notes('geoip_country_name', $name);
return $name;
return $self->{_geoip_record}->country_name();
}
sub set_continent {
my $self = shift;
return $self->set_continent_gc() if $self->{_geoip_city};
my $c_code = shift or return;
my $continent = $self->{_geoip}->continent_code_by_country_code($c_code)
or return;
my ($self, $country_code) = @_;
$country_code or return;
my $continent = $self->{_geoip_city}
? $self->get_continent_gc()
: $self->{_geoip}->continent_code_by_country_code($country_code);
$continent or return;
$self->qp->connection->notes('geoip_continent', $continent);
return $continent;
}
sub set_continent_gc {
sub get_continent_gc {
my $self = shift;
return if !$self->{_geoip_record};
my $continent = $self->{_geoip_record}->continent_code() or return;
$self->qp->connection->notes('geoip_continent', $continent);
return $continent;
return $self->{_geoip_record}->continent_code();
}
sub set_asn {
my ($self, $ip) = @_;
$ip ||= $self->qp->connection->remote_ip;
if (Qpsmtpd::Base->is_ipv6($ip)) {
return $self->set_asn_ipv6($ip);
}
return if ! $self->{GeoIPASNum};
my $asn = $self->{GeoIPASNum}->name_by_addr($ip) or return;
if ('AS' eq substr($asn, 0, 2)) {
$asn = substr($asn, 2);
}
$self->qp->connection->notes('geoip_asn', $asn);
return $asn;
}
sub set_asn_ipv6 {
my ($self, $ip) = @_;
$ip ||= $self->qp->connection->remote_ip;
return if ! $self->{GeoIPASNumv6};
my $asn = $self->{GeoIPASNumv6}->name_by_addr_v6($ip) or return;
$self->qp->connection->notes('geoip_asn', $asn);
return $asn;
}
sub set_city_gc {
my $self = shift;
return if !$self->{_geoip_record};
my $remote_ip = $self->qp->connection->remote_ip;
my $city = $self->{_geoip_record}->city() or return;
$self->qp->connection->notes('geoip_city', $city);
return $city;

71
t/plugin_tests/ident/geoip
View File

@ -3,33 +3,56 @@
use strict;
use warnings;
use lib 'lib';
use Qpsmtpd::Constants;
sub register_tests {
my $self = shift;
eval 'use Geo::IP';
if ( $@ ) {
warn "could not load Geo::IP\n";
return;
};
eval 'use GeoIP2::Database::Reader';
if ( !$@ ) {
warn "using GeoIP2\n";
$self->register_test('test_geoip2_lookup');
}
$self->register_test('test_geoip_lookup');
$self->register_test('test_geoip_load_db');
$self->register_test('test_geoip_init_cc');
$self->register_test('test_set_country_code');
$self->register_test('test_set_country_name');
$self->register_test('test_set_continent');
$self->register_test('test_set_distance');
eval 'use Geo::IP';
if ( !$@ ) {
warn "loaded Geo::IP\n";
$self->register_test('test_geoip_lookup');
$self->register_test('test_geoip_load_db');
$self->register_test('test_geoip_init_cc');
$self->register_test('test_set_country_code');
$self->register_test('test_set_country_name');
$self->register_test('test_set_continent');
$self->register_test('test_set_distance');
$self->register_test('test_set_asn');
};
};
sub test_geoip2_lookup {
my $self = shift;
$self->qp->connection->remote_ip('24.24.24.24');
cmp_ok( $self->geoip2_lookup(), '==', DECLINED, "exit code DECLINED");
if (!$self->load_geoip2()) {
warn "failed to load GeoIP2\n";
}
cmp_ok( $self->connection->notes('geoip_country'), 'eq', 'US', "24.24.24.24 is in country US");
cmp_ok( $self->connection->notes('geoip_country_name'), 'eq', 'United States', "24.24.24.24 is in country United States");
cmp_ok( $self->connection->notes('geoip_continent'), 'eq', 'NA', "24.24.24.24 is in continent NA");
cmp_ok( $self->connection->notes('geoip_city'), 'eq', 'Deer Park', "24.24.24.24 is in city of Deer Park");
};
sub test_geoip_lookup {
my $self = shift;
$self->qp->connection->remote_ip('24.24.24.24');
cmp_ok( $self->connect_handler(), '==', DECLINED, "exit code");
cmp_ok( $self->geoip_lookup(), '==', DECLINED, "exit code");
cmp_ok( $self->connection->notes('geoip_country'), 'eq', 'US', "note");
cmp_ok( $self->connection->notes('geoip_country'), 'eq', 'US', "24.24.24.24 is in the US");
};
sub test_geoip_load_db {
@ -73,10 +96,10 @@ sub test_set_country_code {
$self->qp->connection->remote_ip('24.24.24.24');
$cc = $self->set_country_code();
cmp_ok( $cc, 'eq', 'US', "$cc");
cmp_ok( $cc, 'eq', 'US', "set_country_code result is $cc");
my $note = $self->connection->notes('geoip_country');
cmp_ok( $note, 'eq', 'US', "note has: $cc");
cmp_ok( $note, 'eq', 'US', "set_country_code set note to $cc");
};
sub test_set_country_name {
@ -143,4 +166,20 @@ sub test_set_distance {
ok( 1, "no distance data");
}
};
sub test_set_asn {
my $self = shift;
$self->qp->connection->remote_ip('');
$self->set_asn();
my $asn = $self->set_asn();
ok( ! $asn, "undef") or warn "$asn\n";
$self->qp->connection->remote_ip('24.24.24.24');
$asn = $self->set_asn();
ok( $self->connection->notes('geoip_asn') =~ /^11351/, "note has: $asn");
$self->qp->connection->remote_ip('66.128.51.163');
$asn = $self->set_asn();
ok( $self->connection->notes('geoip_asn') =~ /^7819/, "note has: $asn");
};