From a3b8257b736c4f82a87860a2edb7da03e08b842c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Tue, 7 Jul 2015 17:45:40 +0200 Subject: [PATCH] Add HEAD method verb to apache-badbots, nginx-badbots --- ChangeLog | 2 ++ config/filter.d/apache-badbots.conf | 2 +- config/filter.d/nginx-botsearch.conf | 4 ++-- fail2ban/tests/files/logs/apache-badbots | 6 ++++++ fail2ban/tests/files/logs/nginx-botsearch | 12 ++++++++++++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 82a04f51..ec8fd20b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,8 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released - IMPORTANT incompatible changes: * filter.d/roundcube-auth.conf - Changed logpath to 'errors' log (was 'userlogins') + * filter.d/apache-badbots.conf, filter.d/nginx-botsearch.conf - add + HEAD method verb - Fixes: * reload in interactive mode appends all the jails twice (gh-825) diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf index b2ac9626..75c0de03 100644 --- a/config/filter.d/apache-badbots.conf +++ b/config/filter.d/apache-badbots.conf @@ -10,7 +10,7 @@ badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 +http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, +http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00 -failregex = ^ -.*"(GET|POST).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$ +failregex = ^ -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$ ignoreregex = diff --git a/config/filter.d/nginx-botsearch.conf b/config/filter.d/nginx-botsearch.conf index 567f2f56..6853e1e8 100644 --- a/config/filter.d/nginx-botsearch.conf +++ b/config/filter.d/nginx-botsearch.conf @@ -8,8 +8,8 @@ before = botsearch-common.conf [Definition] -failregex = ^ \- \S+ \[\] \"(GET|POST) \/ \S+\" 404 .+$ - ^ \[error\] \d+#\d+: \*\d+ (\S+ )?\"\S+\" (failed|is not found) \(2\: No such file or directory\), client\: \, server\: \S*\, request: \"(GET|POST) \/ \S+\"\, .*?$ +failregex = ^ \- \S+ \[\] \"(GET|POST|HEAD) \/ \S+\" 404 .+$ + ^ \[error\] \d+#\d+: \*\d+ (\S+ )?\"\S+\" (failed|is not found) \(2\: No such file or directory\), client\: \, server\: \S*\, request: \"(GET|POST|HEAD) \/ \S+\"\, .*?$ ignoreregex = diff --git a/fail2ban/tests/files/logs/apache-badbots b/fail2ban/tests/files/logs/apache-badbots index 35669252..5486f36a 100644 --- a/fail2ban/tests/files/logs/apache-badbots +++ b/fail2ban/tests/files/logs/apache-badbots @@ -1,2 +1,8 @@ # failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" } 1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" + +# failJSON: { "time": "2007-03-05T14:40:21", "match": true , "host": "1.2.3.4" } +1.2.3.4 - - [05/Mar/2007:14:40:21 +0100] "GET /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" + +# failJSON: { "time": "2007-03-05T14:41:21", "match": true , "host": "1.2.3.4" } +1.2.3.4 - - [05/Mar/2007:14:41:21 +0100] "HEAD /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" diff --git a/fail2ban/tests/files/logs/nginx-botsearch b/fail2ban/tests/files/logs/nginx-botsearch index f1bf05f5..c694fa3b 100644 --- a/fail2ban/tests/files/logs/nginx-botsearch +++ b/fail2ban/tests/files/logs/nginx-botsearch @@ -10,6 +10,12 @@ # failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } 12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "GET //admin/pma/scripts/setup.php HTTP/1.1" 404 47 "-" "-" "-" +# failJSON: { "time": "2015-01-20T19:54:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:54:28 +0100] "POST //admin/pma/scripts/setup.php HTTP/1.1" 404 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:55:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:55:28 +0100] "HEAD //admin/pma/scripts/setup.php HTTP/1.1" 404 47 "-" "-" "-" + # failJSON: { "time": "2015-01-20T01:17:07", "match": true , "host": "7.8.9.10" } 7.8.9.10 - root [20/Jan/2015:01:17:07 +0100] "GET /cgi-bin/recent.cgi HTTP/1.1" 404 162 "-" "-" "-" @@ -19,5 +25,11 @@ # failJSON: { "time": "2015-01-21T10:56:10", "match": true , "host": "5.7.9.2" } 2015/01/21 10:56:10 [error] 2833#0: *16336 open() "/var/www/site/cgi-bin/php4" failed (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "GET /cgi-bin/php4 HTTP/1.1", host: "1.2.3.4" +# failJSON: { "time": "2015-01-21T10:57:10", "match": true , "host": "5.7.9.2" } +2015/01/21 10:57:10 [error] 2833#0: *16336 open() "/var/www/site/cgi-bin/php4" failed (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "POST /cgi-bin/php4 HTTP/1.1", host: "1.2.3.4" + +# failJSON: { "time": "2015-01-21T10:58:10", "match": true , "host": "5.7.9.2" } +2015/01/21 10:58:10 [error] 2833#0: *16336 open() "/var/www/site/cgi-bin/php4" failed (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "HEAD /cgi-bin/php4 HTTP/1.1", host: "1.2.3.4" + # failJSON: { "time": "2015-01-21T15:02:27", "match": true , "host": "5.7.9.2" } 2015/01/21 15:02:27 [error] 2833#0: *16813 "/var/www/site/roundcube/" is not found (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "GET /roundcube/ HTTP/1.1", host: "1.2.3.4" \ No newline at end of file