Regex fix, checkip argument & updated tld list (#283)
* Regex fix, checkip argument & updated tld list This patch include 3 fixes: 1) The plugin could match invalid URIs, eg. "www..example.com", which resulted in fatal plugin error, sending the incoming email directly into queue without any further scan. The regex has been fixed. 2) Some URIBLs trigger positive on any ip-address. Added new plugin argument: checkip [yes|no]. 3) Updating the TLD list. Co-authored-by: mufus <hello@mufus.com>
This commit is contained in:
mufus
GitHub
parent
65fd1b2150
commit
e21860f22e
147
plugins/uribl
147
plugins/uribl
|
|
@ -66,6 +66,12 @@ Net::DNS documentation for details.
|
|||
If set true, any headers found in the URIs will be checked as well. Disabled
|
||||
by default.
|
||||
|
||||
=item checkip [yes|no]
|
||||
|
||||
Specifies whether ip-addresses should be searched for or not. Some URIBLs
|
||||
only allows domain names and triggers positive on any ip-address.
|
||||
Default value: yes.
|
||||
|
||||
=back
|
||||
|
||||
=head1 CAUTIONS
|
||||
|
|
@ -139,6 +145,7 @@ sub init {
|
|||
|
||||
$self->{action} = $args{action} || 'add-header';
|
||||
$self->{timeout} = $args{timeout} || 30;
|
||||
$self->{checkip} = $args{checkip} || 'yes';
|
||||
|
||||
# scan-headers was the originally documented name for this option, while
|
||||
# check-headers actually implements it, so tolerate both.
|
||||
|
|
@ -257,6 +264,8 @@ sub lookup_start {
|
|||
my $queries = 0;
|
||||
my %pending;
|
||||
my @qp_continuations;
|
||||
|
||||
my $tlds4regex = $self->get_tlds4regex;
|
||||
|
||||
$transaction->body_resetpos;
|
||||
|
||||
|
|
@ -266,13 +275,15 @@ sub lookup_start {
|
|||
chomp $l;
|
||||
last if !$l;
|
||||
}
|
||||
while ($l = $transaction->body_getline) {
|
||||
while ($l = $transaction->body_getline || @qp_continuations) {
|
||||
chomp $l;
|
||||
|
||||
if ($l =~ /(.*)=$/) {
|
||||
push @qp_continuations, $1;
|
||||
next;
|
||||
}
|
||||
elsif (@qp_continuations) {
|
||||
$l = '' if $l eq '1';
|
||||
$l = join('', @qp_continuations, $l);
|
||||
@qp_continuations = ();
|
||||
}
|
||||
|
|
@ -286,6 +297,8 @@ sub lookup_start {
|
|||
# Dodge inserted-semicolon munging
|
||||
$l =~ tr/;//d;
|
||||
|
||||
# Check for IP-addresses?
|
||||
if ($self->{checkip} eq 'yes') {
|
||||
while (
|
||||
$l =~ m{
|
||||
\w{3,16}:/+ # protocol
|
||||
|
|
@ -338,14 +351,12 @@ sub lookup_start {
|
|||
$pending{$rev} = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
while (
|
||||
$l =~ m{
|
||||
((?:www\.)? # www?
|
||||
[a-zA-Z0-9][a-zA-Z0-9\-.]+\. # hostname
|
||||
(?:aero|arpa|asia|biz|cat|com|coop| # tld
|
||||
edu|gov|info|int|jobs|mil|mobi|
|
||||
museum|name|net|org|pro|tel|travel|
|
||||
[a-zA-Z]{2})
|
||||
[a-zA-Z0-9](?:[a-zA-Z0-9\-]+\.)+ # hostname
|
||||
(?:$tlds4regex|[a-zA-Z]{2}) # tld
|
||||
)(?!\w)
|
||||
}gix
|
||||
)
|
||||
|
|
@ -386,12 +397,9 @@ sub lookup_start {
|
|||
\w{3,16}:/+ # protocol
|
||||
(?:\S+@)? # user/pass
|
||||
(
|
||||
[a-zA-Z0-9][a-zA-Z0-9\-.]+\. # hostname
|
||||
(?:aero|arpa|asia|biz|cat|com|coop| # tld
|
||||
edu|gov|info|int|jobs|mil|mobi|
|
||||
museum|name|net|org|pro|tel|travel|
|
||||
[a-zA-Z]{2})
|
||||
)
|
||||
[a-zA-Z0-9](?:[a-zA-Z0-9\-]+\.)+ # hostname
|
||||
(?:$tlds4regex|[a-zA-Z]{2}) # tld
|
||||
)(?!\w)
|
||||
}gix
|
||||
)
|
||||
{
|
||||
|
|
@ -532,3 +540,118 @@ sub hook_data_post {
|
|||
return DECLINED;
|
||||
}
|
||||
|
||||
sub get_tlds4regex {
|
||||
|
||||
# TLD list from http://data.iana.org/TLD/tlds-alpha-by-domain.txt as of 2017-03-03
|
||||
|
||||
return 'aaa|aarp|abarth|abb|abbott|abbvie|abc|able|abogado|abudhabi|ac|academy|
|
||||
accenture|accountant|accountants|aco|active|actor|ad|adac|ads|adult|ae|aeg|aero|
|
||||
aetna|af|afamilycompany|afl|ag|agakhan|agency|ai|aig|aigo|airbus|airforce|
|
||||
airtel|akdn|al|alfaromeo|alibaba|alipay|allfinanz|allstate|ally|alsace|alstom|
|
||||
am|americanexpress|americanfamily|amex|amfam|amica|amsterdam|analytics|android|
|
||||
anquan|anz|ao|aol|apartments|app|apple|aq|aquarelle|ar|aramco|archi|army|arpa|
|
||||
art|arte|as|asda|asia|associates|at|athleta|attorney|au|auction|audi|audible|
|
||||
audio|auspost|author|auto|autos|avianca|aw|aws|ax|axa|az|azure|ba|baby|baidu|
|
||||
banamex|bananarepublic|band|bank|bar|barcelona|barclaycard|barclays|barefoot|
|
||||
bargains|baseball|basketball|bauhaus|bayern|bb|bbc|bbt|bbva|bcg|bcn|bd|be|beats|
|
||||
beauty|beer|bentley|berlin|best|bestbuy|bet|bf|bg|bh|bharti|bi|bible|bid|bike|
|
||||
bing|bingo|bio|biz|bj|black|blackfriday|blanco|blockbuster|blog|bloomberg|blue|
|
||||
bm|bms|bmw|bn|bnl|bnpparibas|bo|boats|boehringer|bofa|bom|bond|boo|book|booking|
|
||||
boots|bosch|bostik|bot|boutique|br|bradesco|bridgestone|broadway|broker|brother|
|
||||
brussels|bs|bt|budapest|bugatti|build|builders|business|buy|buzz|bv|bw|by|bz|
|
||||
bzh|ca|cab|cafe|cal|call|calvinklein|cam|camera|camp|cancerresearch|canon|
|
||||
capetown|capital|capitalone|car|caravan|cards|care|career|careers|cars|cartier|
|
||||
casa|case|caseih|cash|casino|cat|catering|cba|cbn|cbre|cbs|cc|cd|ceb|center|ceo|
|
||||
cern|cf|cfa|cfd|cg|ch|chanel|channel|chase|chat|cheap|chintai|chloe|christmas|
|
||||
chrome|chrysler|church|ci|cipriani|circle|cisco|citadel|citi|citic|city|
|
||||
cityeats|ck|cl|claims|cleaning|click|clinic|clinique|clothing|cloud|club|
|
||||
clubmed|cm|cn|co|coach|codes|coffee|college|cologne|com|comcast|commbank|
|
||||
community|company|compare|computer|comsec|condos|construction|consulting|
|
||||
contact|contractors|cooking|cookingchannel|cool|coop|corsica|country|coupon|
|
||||
coupons|courses|cr|credit|creditcard|creditunion|cricket|crown|crs|cruises|csc|
|
||||
cu|cuisinella|cv|cw|cx|cy|cymru|cyou|cz|dabur|dad|dance|date|dating|datsun|day|
|
||||
dclk|dds|de|deal|dealer|deals|degree|delivery|dell|deloitte|delta|democrat|
|
||||
dental|dentist|desi|design|dev|dhl|diamonds|diet|digital|direct|directory|
|
||||
discount|discover|dish|diy|dj|dk|dm|dnp|do|docs|doctor|dodge|dog|doha|domains|
|
||||
dot|download|drive|dtv|dubai|duck|dunlop|duns|dupont|durban|dvag|dvr|dz|earth|
|
||||
eat|ec|eco|edeka|edu|education|ee|eg|email|emerck|energy|engineer|engineering|
|
||||
enterprises|epost|epson|equipment|er|ericsson|erni|es|esq|estate|esurance|et|eu|
|
||||
eurovision|eus|events|everbank|exchange|expert|exposed|express|extraspace|fage|
|
||||
fail|fairwinds|faith|family|fan|fans|farm|farmers|fashion|fast|fedex|feedback|
|
||||
ferrari|ferrero|fi|fiat|fidelity|fido|film|final|finance|financial|fire|
|
||||
firestone|firmdale|fish|fishing|fit|fitness|fj|fk|flickr|flights|flir|florist|
|
||||
flowers|fly|fm|fo|foo|foodnetwork|football|ford|forex|forsale|forum|foundation|
|
||||
fox|fr|free|fresenius|frl|frogans|frontdoor|frontier|ftr|fujitsu|fujixerox|fund|
|
||||
furniture|futbol|fyi|ga|gal|gallery|gallo|gallup|game|games|gap|garden|gb|gbiz|
|
||||
gd|gdn|ge|gea|gent|genting|george|gf|gg|ggee|gh|gi|gift|gifts|gives|giving|gl|
|
||||
glade|glass|gle|global|globo|gm|gmail|gmbh|gmo|gmx|gn|godaddy|gold|goldpoint|
|
||||
golf|goo|goodhands|goodyear|goog|google|gop|got|gov|gp|gq|gr|grainger|graphics|
|
||||
gratis|green|gripe|group|gs|gt|gu|guardian|gucci|guge|guide|guitars|guru|gw|gy|
|
||||
hamburg|hangout|haus|hbo|hdfc|hdfcbank|health|healthcare|help|helsinki|here|
|
||||
hermes|hgtv|hiphop|hisamitsu|hitachi|hiv|hk|hkt|hm|hn|hockey|holdings|holiday|
|
||||
homedepot|homegoods|homes|homesense|honda|honeywell|horse|host|hosting|hot|
|
||||
hoteles|hotmail|house|how|hr|hsbc|ht|htc|hu|hughes|hyatt|hyundai|ibm|icbc|ice|
|
||||
icu|id|ie|ieee|ifm|iinet|ikano|il|im|imamat|imdb|immo|immobilien|in|industries|
|
||||
infiniti|info|ing|ink|institute|insurance|insure|int|intel|international|intuit|
|
||||
investments|io|ipiranga|iq|ir|irish|is|iselect|ismaili|ist|istanbul|it|itau|itv|
|
||||
iveco|iwc|jaguar|java|jcb|jcp|je|jeep|jetzt|jewelry|jlc|jll|jm|jmp|jnj|jo|jobs|
|
||||
joburg|jot|joy|jp|jpmorgan|jprs|juegos|juniper|kaufen|kddi|ke|kerryhotels|
|
||||
kerrylogistics|kerryproperties|kfh|kg|kh|ki|kia|kim|kinder|kindle|kitchen|kiwi|
|
||||
km|kn|koeln|komatsu|kosher|kp|kpmg|kpn|kr|krd|kred|kuokgroup|kw|ky|kyoto|kz|la|
|
||||
lacaixa|ladbrokes|lamborghini|lamer|lancaster|lancia|lancome|land|landrover|
|
||||
lanxess|lasalle|lat|latino|latrobe|law|lawyer|lb|lc|lds|lease|leclerc|lefrak|
|
||||
legal|lego|lexus|lgbt|li|liaison|lidl|life|lifeinsurance|lifestyle|lighting|
|
||||
like|lilly|limited|limo|lincoln|linde|link|lipsy|live|living|lixil|lk|loan|
|
||||
loans|locker|locus|loft|lol|london|lotte|lotto|love|lpl|lplfinancial|lr|ls|lt|
|
||||
ltd|ltda|lu|lundbeck|lupin|luxe|luxury|lv|ly|ma|macys|madrid|maif|maison|makeup|
|
||||
man|management|mango|market|marketing|markets|marriott|marshalls|maserati|
|
||||
mattel|mba|mc|mcd|mcdonalds|mckinsey|md|me|med|media|meet|melbourne|meme|
|
||||
memorial|men|menu|meo|metlife|mg|mh|miami|microsoft|mil|mini|mint|mit|
|
||||
mitsubishi|mk|ml|mlb|mls|mm|mma|mn|mo|mobi|mobily|moda|moe|moi|mom|monash|money|
|
||||
monster|montblanc|mopar|mormon|mortgage|moscow|motorcycles|mov|movie|movistar|
|
||||
mp|mq|mr|ms|msd|mt|mtn|mtpc|mtr|mu|museum|mutual|mutuelle|mv|mw|mx|my|mz|na|nab|
|
||||
nadex|nagoya|name|nationwide|natura|navy|nba|nc|ne|nec|net|netbank|netflix|
|
||||
network|neustar|new|newholland|news|next|nextdirect|nexus|nf|nfl|ng|ngo|nhk|ni|
|
||||
nico|nike|nikon|ninja|nissan|nissay|nl|no|nokia|northwesternmutual|norton|now|
|
||||
nowruz|nowtv|np|nr|nra|nrw|ntt|nu|nyc|nz|obi|observer|off|office|okinawa|olayan|
|
||||
olayangroup|oldnavy|ollo|om|omega|one|ong|onl|online|onyourside|ooo|open|oracle|
|
||||
orange|org|organic|orientexpress|origins|osaka|otsuka|ott|ovh|pa|page|
|
||||
pamperedchef|panasonic|panerai|paris|pars|partners|parts|party|passagens|pay|
|
||||
pccw|pe|pet|pf|pfizer|pg|ph|pharmacy|philips|photo|photography|photos|physio|
|
||||
piaget|pics|pictet|pictures|pid|pin|ping|pink|pioneer|pizza|pk|pl|place|play|
|
||||
playstation|plumbing|plus|pm|pn|pnc|pohl|poker|politie|porn|post|pr|pramerica|
|
||||
praxi|press|prime|pro|prod|productions|prof|progressive|promo|properties|
|
||||
property|protection|pru|prudential|ps|pt|pub|pw|pwc|py|qa|qpon|quebec|quest|qvc|
|
||||
racing|radio|raid|re|read|realestate|realtor|realty|recipes|red|redstone|
|
||||
redumbrella|rehab|reise|reisen|reit|ren|rent|rentals|repair|report|republican|
|
||||
rest|restaurant|review|reviews|rexroth|rich|richardli|ricoh|rightathome|rio|rip|
|
||||
ro|rocher|rocks|rodeo|rogers|room|rs|rsvp|ru|ruhr|run|rw|rwe|ryukyu|sa|saarland|
|
||||
safe|safety|sakura|sale|salon|samsclub|samsung|sandvik|sandvikcoromant|sanofi|
|
||||
sap|sapo|sarl|sas|save|saxo|sb|sbi|sbs|sc|sca|scb|schaeffler|schmidt|
|
||||
scholarships|school|schule|schwarz|science|scjohnson|scor|scot|sd|se|seat|
|
||||
secure|security|seek|select|sener|services|ses|seven|sew|sex|sexy|sfr|sg|sh|
|
||||
shangrila|sharp|shaw|shell|shia|shiksha|shoes|shop|shopping|shouji|show|
|
||||
showtime|shriram|si|silk|sina|singles|site|sj|sk|ski|skin|sky|skype|sl|sling|sm|
|
||||
smart|smile|sn|sncf|so|soccer|social|softbank|software|sohu|solar|solutions|
|
||||
song|sony|soy|space|spiegel|spot|spreadbetting|sr|srl|srt|st|stada|staples|star|
|
||||
starhub|statebank|statefarm|statoil|stc|stcgroup|stockholm|storage|store|stream|
|
||||
studio|study|style|su|sucks|supplies|supply|support|surf|surgery|suzuki|sv|
|
||||
swatch|swiftcover|swiss|sx|sy|sydney|symantec|systems|sz|tab|taipei|talk|taobao|
|
||||
target|tatamotors|tatar|tattoo|tax|taxi|tc|tci|td|tdk|team|tech|technology|tel|
|
||||
telecity|telefonica|temasek|tennis|teva|tf|tg|th|thd|theater|theatre|tiaa|
|
||||
tickets|tienda|tiffany|tips|tires|tirol|tj|tjmaxx|tjx|tk|tkmaxx|tl|tm|tmall|tn|
|
||||
to|today|tokyo|tools|top|toray|toshiba|total|tours|town|toyota|toys|tr|trade|
|
||||
trading|training|travel|travelchannel|travelers|travelersinsurance|trust|trv|tt|
|
||||
tube|tui|tunes|tushu|tv|tvs|tw|tz|ua|ubank|ubs|uconnect|ug|uk|unicom|university|
|
||||
uno|uol|ups|us|uy|uz|va|vacations|vana|vanguard|vc|ve|vegas|ventures|verisign|
|
||||
versicherung|vet|vg|vi|viajes|video|vig|viking|villas|vin|vip|virgin|visa|
|
||||
vision|vista|vistaprint|viva|vivo|vlaanderen|vn|vodka|volkswagen|volvo|vote|
|
||||
voting|voto|voyage|vu|vuelos|wales|walmart|walter|wang|wanggou|warman|watch|
|
||||
watches|weather|weatherchannel|webcam|weber|website|wed|wedding|weibo|weir|wf|
|
||||
whoswho|wien|wiki|williamhill|win|windows|wine|winners|wme|wolterskluwer|
|
||||
woodside|work|works|world|wow|ws|wtc|wtf|xbox|xerox|xfinity|xihuan|xin|xperia|
|
||||
xxx|xyz|yachts|yahoo|yamaxun|yandex|ye|yodobashi|yoga|yokohama|you|youtube|yt|
|
||||
yun|za|zappos|zara|zero|zip|zippo|zm|zone|zuerich|zw';
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue