Regex fix, checkip argument & updated tld list (#283)
* Regex fix, checkip argument & updated tld list This patch include 3 fixes: 1) The plugin could match invalid URIs, eg. "www..example.com", which resulted in fatal plugin error, sending the incoming email directly into queue without any further scan. The regex has been fixed. 2) Some URIBLs trigger positive on any ip-address. Added new plugin argument: checkip [yes|no]. 3) Updating the TLD list. Co-authored-by: mufus <hello@mufus.com>
This commit is contained in:
mufus
GitHub
parent
65fd1b2150
commit
e21860f22e
147
plugins/uribl
147
plugins/uribl
@ -66,6 +66,12 @@ Net::DNS documentation for details.
|
|||||||
If set true, any headers found in the URIs will be checked as well. Disabled
|
If set true, any headers found in the URIs will be checked as well. Disabled
|
||||||
by default.
|
by default.
|
||||||
|
|
||||||
|
=item checkip [yes|no]
|
||||||
|
|
||||||
|
Specifies whether ip-addresses should be searched for or not. Some URIBLs
|
||||||
|
only allows domain names and triggers positive on any ip-address.
|
||||||
|
Default value: yes.
|
||||||
|
|
||||||
=back
|
=back
|
||||||
|
|
||||||
=head1 CAUTIONS
|
=head1 CAUTIONS
|
||||||
@ -139,6 +145,7 @@ sub init {
|
|||||||
|
|
||||||
$self->{action} = $args{action} || 'add-header';
|
$self->{action} = $args{action} || 'add-header';
|
||||||
$self->{timeout} = $args{timeout} || 30;
|
$self->{timeout} = $args{timeout} || 30;
|
||||||
|
$self->{checkip} = $args{checkip} || 'yes';
|
||||||
|
|
||||||
# scan-headers was the originally documented name for this option, while
|
# scan-headers was the originally documented name for this option, while
|
||||||
# check-headers actually implements it, so tolerate both.
|
# check-headers actually implements it, so tolerate both.
|
||||||
@ -258,6 +265,8 @@ sub lookup_start {
|
|||||||
my %pending;
|
my %pending;
|
||||||
my @qp_continuations;
|
my @qp_continuations;
|
||||||
|
|
||||||
|
my $tlds4regex = $self->get_tlds4regex;
|
||||||
|
|
||||||
$transaction->body_resetpos;
|
$transaction->body_resetpos;
|
||||||
|
|
||||||
# if we're not looking for URIs in the headers, read past that point
|
# if we're not looking for URIs in the headers, read past that point
|
||||||
@ -266,13 +275,15 @@ sub lookup_start {
|
|||||||
chomp $l;
|
chomp $l;
|
||||||
last if !$l;
|
last if !$l;
|
||||||
}
|
}
|
||||||
while ($l = $transaction->body_getline) {
|
while ($l = $transaction->body_getline || @qp_continuations) {
|
||||||
chomp $l;
|
chomp $l;
|
||||||
|
|
||||||
if ($l =~ /(.*)=$/) {
|
if ($l =~ /(.*)=$/) {
|
||||||
push @qp_continuations, $1;
|
push @qp_continuations, $1;
|
||||||
|
next;
|
||||||
}
|
}
|
||||||
elsif (@qp_continuations) {
|
elsif (@qp_continuations) {
|
||||||
|
$l = '' if $l eq '1';
|
||||||
$l = join('', @qp_continuations, $l);
|
$l = join('', @qp_continuations, $l);
|
||||||
@qp_continuations = ();
|
@qp_continuations = ();
|
||||||
}
|
}
|
||||||
@ -286,6 +297,8 @@ sub lookup_start {
|
|||||||
# Dodge inserted-semicolon munging
|
# Dodge inserted-semicolon munging
|
||||||
$l =~ tr/;//d;
|
$l =~ tr/;//d;
|
||||||
|
|
||||||
|
# Check for IP-addresses?
|
||||||
|
if ($self->{checkip} eq 'yes') {
|
||||||
while (
|
while (
|
||||||
$l =~ m{
|
$l =~ m{
|
||||||
\w{3,16}:/+ # protocol
|
\w{3,16}:/+ # protocol
|
||||||
@ -338,14 +351,12 @@ sub lookup_start {
|
|||||||
$pending{$rev} = 1;
|
$pending{$rev} = 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
while (
|
while (
|
||||||
$l =~ m{
|
$l =~ m{
|
||||||
((?:www\.)? # www?
|
((?:www\.)? # www?
|
||||||
[a-zA-Z0-9][a-zA-Z0-9\-.]+\. # hostname
|
[a-zA-Z0-9](?:[a-zA-Z0-9\-]+\.)+ # hostname
|
||||||
(?:aero|arpa|asia|biz|cat|com|coop| # tld
|
(?:$tlds4regex|[a-zA-Z]{2}) # tld
|
||||||
edu|gov|info|int|jobs|mil|mobi|
|
|
||||||
museum|name|net|org|pro|tel|travel|
|
|
||||||
[a-zA-Z]{2})
|
|
||||||
)(?!\w)
|
)(?!\w)
|
||||||
}gix
|
}gix
|
||||||
)
|
)
|
||||||
@ -386,12 +397,9 @@ sub lookup_start {
|
|||||||
\w{3,16}:/+ # protocol
|
\w{3,16}:/+ # protocol
|
||||||
(?:\S+@)? # user/pass
|
(?:\S+@)? # user/pass
|
||||||
(
|
(
|
||||||
[a-zA-Z0-9][a-zA-Z0-9\-.]+\. # hostname
|
[a-zA-Z0-9](?:[a-zA-Z0-9\-]+\.)+ # hostname
|
||||||
(?:aero|arpa|asia|biz|cat|com|coop| # tld
|
(?:$tlds4regex|[a-zA-Z]{2}) # tld
|
||||||
edu|gov|info|int|jobs|mil|mobi|
|
)(?!\w)
|
||||||
museum|name|net|org|pro|tel|travel|
|
|
||||||
[a-zA-Z]{2})
|
|
||||||
)
|
|
||||||
}gix
|
}gix
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
@ -532,3 +540,118 @@ sub hook_data_post {
|
|||||||
return DECLINED;
|
return DECLINED;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sub get_tlds4regex {
|
||||||
|
|
||||||
|
# TLD list from http://data.iana.org/TLD/tlds-alpha-by-domain.txt as of 2017-03-03
|
||||||
|
|
||||||
|
return 'aaa|aarp|abarth|abb|abbott|abbvie|abc|able|abogado|abudhabi|ac|academy|
|
||||||
|
accenture|accountant|accountants|aco|active|actor|ad|adac|ads|adult|ae|aeg|aero|
|
||||||
|
aetna|af|afamilycompany|afl|ag|agakhan|agency|ai|aig|aigo|airbus|airforce|
|
||||||
|
airtel|akdn|al|alfaromeo|alibaba|alipay|allfinanz|allstate|ally|alsace|alstom|
|
||||||
|
am|americanexpress|americanfamily|amex|amfam|amica|amsterdam|analytics|android|
|
||||||
|
anquan|anz|ao|aol|apartments|app|apple|aq|aquarelle|ar|aramco|archi|army|arpa|
|
||||||
|
art|arte|as|asda|asia|associates|at|athleta|attorney|au|auction|audi|audible|
|
||||||
|
audio|auspost|author|auto|autos|avianca|aw|aws|ax|axa|az|azure|ba|baby|baidu|
|
||||||
|
banamex|bananarepublic|band|bank|bar|barcelona|barclaycard|barclays|barefoot|
|
||||||
|
bargains|baseball|basketball|bauhaus|bayern|bb|bbc|bbt|bbva|bcg|bcn|bd|be|beats|
|
||||||
|
beauty|beer|bentley|berlin|best|bestbuy|bet|bf|bg|bh|bharti|bi|bible|bid|bike|
|
||||||
|
bing|bingo|bio|biz|bj|black|blackfriday|blanco|blockbuster|blog|bloomberg|blue|
|
||||||
|
bm|bms|bmw|bn|bnl|bnpparibas|bo|boats|boehringer|bofa|bom|bond|boo|book|booking|
|
||||||
|
boots|bosch|bostik|bot|boutique|br|bradesco|bridgestone|broadway|broker|brother|
|
||||||
|
brussels|bs|bt|budapest|bugatti|build|builders|business|buy|buzz|bv|bw|by|bz|
|
||||||
|
bzh|ca|cab|cafe|cal|call|calvinklein|cam|camera|camp|cancerresearch|canon|
|
||||||
|
capetown|capital|capitalone|car|caravan|cards|care|career|careers|cars|cartier|
|
||||||
|
casa|case|caseih|cash|casino|cat|catering|cba|cbn|cbre|cbs|cc|cd|ceb|center|ceo|
|
||||||
|
cern|cf|cfa|cfd|cg|ch|chanel|channel|chase|chat|cheap|chintai|chloe|christmas|
|
||||||
|
chrome|chrysler|church|ci|cipriani|circle|cisco|citadel|citi|citic|city|
|
||||||
|
cityeats|ck|cl|claims|cleaning|click|clinic|clinique|clothing|cloud|club|
|
||||||
|
clubmed|cm|cn|co|coach|codes|coffee|college|cologne|com|comcast|commbank|
|
||||||
|
community|company|compare|computer|comsec|condos|construction|consulting|
|
||||||
|
contact|contractors|cooking|cookingchannel|cool|coop|corsica|country|coupon|
|
||||||
|
coupons|courses|cr|credit|creditcard|creditunion|cricket|crown|crs|cruises|csc|
|
||||||
|
cu|cuisinella|cv|cw|cx|cy|cymru|cyou|cz|dabur|dad|dance|date|dating|datsun|day|
|
||||||
|
dclk|dds|de|deal|dealer|deals|degree|delivery|dell|deloitte|delta|democrat|
|
||||||
|
dental|dentist|desi|design|dev|dhl|diamonds|diet|digital|direct|directory|
|
||||||
|
discount|discover|dish|diy|dj|dk|dm|dnp|do|docs|doctor|dodge|dog|doha|domains|
|
||||||
|
dot|download|drive|dtv|dubai|duck|dunlop|duns|dupont|durban|dvag|dvr|dz|earth|
|
||||||
|
eat|ec|eco|edeka|edu|education|ee|eg|email|emerck|energy|engineer|engineering|
|
||||||
|
enterprises|epost|epson|equipment|er|ericsson|erni|es|esq|estate|esurance|et|eu|
|
||||||
|
eurovision|eus|events|everbank|exchange|expert|exposed|express|extraspace|fage|
|
||||||
|
fail|fairwinds|faith|family|fan|fans|farm|farmers|fashion|fast|fedex|feedback|
|
||||||
|
ferrari|ferrero|fi|fiat|fidelity|fido|film|final|finance|financial|fire|
|
||||||
|
firestone|firmdale|fish|fishing|fit|fitness|fj|fk|flickr|flights|flir|florist|
|
||||||
|
flowers|fly|fm|fo|foo|foodnetwork|football|ford|forex|forsale|forum|foundation|
|
||||||
|
fox|fr|free|fresenius|frl|frogans|frontdoor|frontier|ftr|fujitsu|fujixerox|fund|
|
||||||
|
furniture|futbol|fyi|ga|gal|gallery|gallo|gallup|game|games|gap|garden|gb|gbiz|
|
||||||
|
gd|gdn|ge|gea|gent|genting|george|gf|gg|ggee|gh|gi|gift|gifts|gives|giving|gl|
|
||||||
|
glade|glass|gle|global|globo|gm|gmail|gmbh|gmo|gmx|gn|godaddy|gold|goldpoint|
|
||||||
|
golf|goo|goodhands|goodyear|goog|google|gop|got|gov|gp|gq|gr|grainger|graphics|
|
||||||
|
gratis|green|gripe|group|gs|gt|gu|guardian|gucci|guge|guide|guitars|guru|gw|gy|
|
||||||
|
hamburg|hangout|haus|hbo|hdfc|hdfcbank|health|healthcare|help|helsinki|here|
|
||||||
|
hermes|hgtv|hiphop|hisamitsu|hitachi|hiv|hk|hkt|hm|hn|hockey|holdings|holiday|
|
||||||
|
homedepot|homegoods|homes|homesense|honda|honeywell|horse|host|hosting|hot|
|
||||||
|
hoteles|hotmail|house|how|hr|hsbc|ht|htc|hu|hughes|hyatt|hyundai|ibm|icbc|ice|
|
||||||
|
icu|id|ie|ieee|ifm|iinet|ikano|il|im|imamat|imdb|immo|immobilien|in|industries|
|
||||||
|
infiniti|info|ing|ink|institute|insurance|insure|int|intel|international|intuit|
|
||||||
|
investments|io|ipiranga|iq|ir|irish|is|iselect|ismaili|ist|istanbul|it|itau|itv|
|
||||||
|
iveco|iwc|jaguar|java|jcb|jcp|je|jeep|jetzt|jewelry|jlc|jll|jm|jmp|jnj|jo|jobs|
|
||||||
|
joburg|jot|joy|jp|jpmorgan|jprs|juegos|juniper|kaufen|kddi|ke|kerryhotels|
|
||||||
|
kerrylogistics|kerryproperties|kfh|kg|kh|ki|kia|kim|kinder|kindle|kitchen|kiwi|
|
||||||
|
km|kn|koeln|komatsu|kosher|kp|kpmg|kpn|kr|krd|kred|kuokgroup|kw|ky|kyoto|kz|la|
|
||||||
|
lacaixa|ladbrokes|lamborghini|lamer|lancaster|lancia|lancome|land|landrover|
|
||||||
|
lanxess|lasalle|lat|latino|latrobe|law|lawyer|lb|lc|lds|lease|leclerc|lefrak|
|
||||||
|
legal|lego|lexus|lgbt|li|liaison|lidl|life|lifeinsurance|lifestyle|lighting|
|
||||||
|
like|lilly|limited|limo|lincoln|linde|link|lipsy|live|living|lixil|lk|loan|
|
||||||
|
loans|locker|locus|loft|lol|london|lotte|lotto|love|lpl|lplfinancial|lr|ls|lt|
|
||||||
|
ltd|ltda|lu|lundbeck|lupin|luxe|luxury|lv|ly|ma|macys|madrid|maif|maison|makeup|
|
||||||
|
man|management|mango|market|marketing|markets|marriott|marshalls|maserati|
|
||||||
|
mattel|mba|mc|mcd|mcdonalds|mckinsey|md|me|med|media|meet|melbourne|meme|
|
||||||
|
memorial|men|menu|meo|metlife|mg|mh|miami|microsoft|mil|mini|mint|mit|
|
||||||
|
mitsubishi|mk|ml|mlb|mls|mm|mma|mn|mo|mobi|mobily|moda|moe|moi|mom|monash|money|
|
||||||
|
monster|montblanc|mopar|mormon|mortgage|moscow|motorcycles|mov|movie|movistar|
|
||||||
|
mp|mq|mr|ms|msd|mt|mtn|mtpc|mtr|mu|museum|mutual|mutuelle|mv|mw|mx|my|mz|na|nab|
|
||||||
|
nadex|nagoya|name|nationwide|natura|navy|nba|nc|ne|nec|net|netbank|netflix|
|
||||||
|
network|neustar|new|newholland|news|next|nextdirect|nexus|nf|nfl|ng|ngo|nhk|ni|
|
||||||
|
nico|nike|nikon|ninja|nissan|nissay|nl|no|nokia|northwesternmutual|norton|now|
|
||||||
|
nowruz|nowtv|np|nr|nra|nrw|ntt|nu|nyc|nz|obi|observer|off|office|okinawa|olayan|
|
||||||
|
olayangroup|oldnavy|ollo|om|omega|one|ong|onl|online|onyourside|ooo|open|oracle|
|
||||||
|
orange|org|organic|orientexpress|origins|osaka|otsuka|ott|ovh|pa|page|
|
||||||
|
pamperedchef|panasonic|panerai|paris|pars|partners|parts|party|passagens|pay|
|
||||||
|
pccw|pe|pet|pf|pfizer|pg|ph|pharmacy|philips|photo|photography|photos|physio|
|
||||||
|
piaget|pics|pictet|pictures|pid|pin|ping|pink|pioneer|pizza|pk|pl|place|play|
|
||||||
|
playstation|plumbing|plus|pm|pn|pnc|pohl|poker|politie|porn|post|pr|pramerica|
|
||||||
|
praxi|press|prime|pro|prod|productions|prof|progressive|promo|properties|
|
||||||
|
property|protection|pru|prudential|ps|pt|pub|pw|pwc|py|qa|qpon|quebec|quest|qvc|
|
||||||
|
racing|radio|raid|re|read|realestate|realtor|realty|recipes|red|redstone|
|
||||||
|
redumbrella|rehab|reise|reisen|reit|ren|rent|rentals|repair|report|republican|
|
||||||
|
rest|restaurant|review|reviews|rexroth|rich|richardli|ricoh|rightathome|rio|rip|
|
||||||
|
ro|rocher|rocks|rodeo|rogers|room|rs|rsvp|ru|ruhr|run|rw|rwe|ryukyu|sa|saarland|
|
||||||
|
safe|safety|sakura|sale|salon|samsclub|samsung|sandvik|sandvikcoromant|sanofi|
|
||||||
|
sap|sapo|sarl|sas|save|saxo|sb|sbi|sbs|sc|sca|scb|schaeffler|schmidt|
|
||||||
|
scholarships|school|schule|schwarz|science|scjohnson|scor|scot|sd|se|seat|
|
||||||
|
secure|security|seek|select|sener|services|ses|seven|sew|sex|sexy|sfr|sg|sh|
|
||||||
|
shangrila|sharp|shaw|shell|shia|shiksha|shoes|shop|shopping|shouji|show|
|
||||||
|
showtime|shriram|si|silk|sina|singles|site|sj|sk|ski|skin|sky|skype|sl|sling|sm|
|
||||||
|
smart|smile|sn|sncf|so|soccer|social|softbank|software|sohu|solar|solutions|
|
||||||
|
song|sony|soy|space|spiegel|spot|spreadbetting|sr|srl|srt|st|stada|staples|star|
|
||||||
|
starhub|statebank|statefarm|statoil|stc|stcgroup|stockholm|storage|store|stream|
|
||||||
|
studio|study|style|su|sucks|supplies|supply|support|surf|surgery|suzuki|sv|
|
||||||
|
swatch|swiftcover|swiss|sx|sy|sydney|symantec|systems|sz|tab|taipei|talk|taobao|
|
||||||
|
target|tatamotors|tatar|tattoo|tax|taxi|tc|tci|td|tdk|team|tech|technology|tel|
|
||||||
|
telecity|telefonica|temasek|tennis|teva|tf|tg|th|thd|theater|theatre|tiaa|
|
||||||
|
tickets|tienda|tiffany|tips|tires|tirol|tj|tjmaxx|tjx|tk|tkmaxx|tl|tm|tmall|tn|
|
||||||
|
to|today|tokyo|tools|top|toray|toshiba|total|tours|town|toyota|toys|tr|trade|
|
||||||
|
trading|training|travel|travelchannel|travelers|travelersinsurance|trust|trv|tt|
|
||||||
|
tube|tui|tunes|tushu|tv|tvs|tw|tz|ua|ubank|ubs|uconnect|ug|uk|unicom|university|
|
||||||
|
uno|uol|ups|us|uy|uz|va|vacations|vana|vanguard|vc|ve|vegas|ventures|verisign|
|
||||||
|
versicherung|vet|vg|vi|viajes|video|vig|viking|villas|vin|vip|virgin|visa|
|
||||||
|
vision|vista|vistaprint|viva|vivo|vlaanderen|vn|vodka|volkswagen|volvo|vote|
|
||||||
|
voting|voto|voyage|vu|vuelos|wales|walmart|walter|wang|wanggou|warman|watch|
|
||||||
|
watches|weather|weatherchannel|webcam|weber|website|wed|wedding|weibo|weir|wf|
|
||||||
|
whoswho|wien|wiki|williamhill|win|windows|wine|winners|wme|wolterskluwer|
|
||||||
|
woodside|work|works|world|wow|ws|wtc|wtf|xbox|xerox|xfinity|xihuan|xin|xperia|
|
||||||
|
xxx|xyz|yachts|yahoo|yamaxun|yandex|ye|yodobashi|yoga|yokohama|you|youtube|yt|
|
||||||
|
yun|za|zappos|zara|zero|zip|zippo|zm|zone|zuerich|zw';
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user